From 6bd4312840f2ba10143c9aef598226d837b2234d Mon Sep 17 00:00:00 2001 From: Mateusz Wejman Date: Fri, 19 Apr 2024 09:01:46 +0200 Subject: [PATCH 1/6] Created assoc behaviour for PersistentHashMap --- backend/runtime/PersistentHashMap.c | 568 ++++++++++++++++++++++++++++ backend/runtime/PersistentHashMap.h | 149 ++++++++ 2 files changed, 717 insertions(+) create mode 100644 backend/runtime/PersistentHashMap.c create mode 100644 backend/runtime/PersistentHashMap.h diff --git a/backend/runtime/PersistentHashMap.c b/backend/runtime/PersistentHashMap.c new file mode 100644 index 0000000..335c8ee --- /dev/null +++ b/backend/runtime/PersistentHashMap.c @@ -0,0 +1,568 @@ +#include "Object.h" +#include "PersistentArrayMap.h" +#include "Nil.h" +#include +#include "defines.h" + +static PersistentArrayMap *EMPTY = NULL; + +/* mem done */ +PersistentArrayMap *PersistentArrayMap_empty() { + if (EMPTY == NULL) EMPTY = PersistentArrayMap_create(); + retain(EMPTY); + return EMPTY; +} + +/* mem done */ +PersistentArrayMap *PersistentArrayMap_create() { + size_t size = sizeof(Object) + sizeof(PersistentArrayMap); + Object * super = allocate(size); + PersistentArrayMap *self = Object_data(super); + memset(super, 0, size); + Object_create(super, persistentArrayMapType); + return self; +} + +/* outside refcount system */ +PersistentArrayMap *PersistentArrayMap_copy(PersistentArrayMap *other) { + size_t baseSize = sizeof(PersistentArrayMap); + size_t size = baseSize + sizeof(Object); + Object * super = allocate(size); + PersistentArrayMap *self = Object_data(super); + memcpy(self, other, baseSize); + Object_create(super, persistentArrayMapType); + return self; +} + +/* mem done */ +PersistentArrayMap *PersistentArrayMap_createMany(uint64_t pairCount, ...) { + assert(pairCount < HASHTABLE_THRESHOLD + 1 && "Maps of size > 8 not supported yet"); + va_list args; + va_start(args, pairCount); + + PersistentArrayMap *self = PersistentArrayMap_create(); + + self->count = pairCount; + for (int i = 0; i < pairCount; i++) { + void *k = va_arg(args, + void *); + void *v = va_arg(args, + void *); + self->keys[i] = k; + self->values[i] = v; + } + va_end(args); + return self; +} + +/* outside refcount system */ +BOOL PersistentArrayMap_equals(PersistentArrayMap *self, PersistentArrayMap *other) { + if (self->count != other->count) return FALSE; + for (int i = 0; i < self->count; i++) { + void *key = self->keys[i]; + void *val = self->values[i]; + retain(self); + retain(key); + void *otherVal = PersistentArrayMap_get(other, key); + if (!equals(otherVal, val)) return FALSE; + } + return TRUE; +} + +/* outside refcount system */ +uint64_t PersistentArrayMap_hash(PersistentArrayMap *self) { + uint64_t h = 5381; + for (int i = 0; i < self->count; i++) { + void *key = self->keys[i]; + void *value = self->values[i]; + h += (hash(key) ^ hash(value)); + } + return h; +} + +/* mem done */ +String *PersistentArrayMap_toString(PersistentArrayMap *self) { + String * retVal = String_create("{"); + String * space = String_create(" "); + String * comma = String_create(", "); + String * closing = String_create("}"); + + BOOL hasAtLeastOne = FALSE; + for (int i = 0; i < self->count; i++) { + void *key = self->keys[i]; + if (hasAtLeastOne) { + retain(comma); + retVal = String_concat(retVal, comma); + } + hasAtLeastOne = TRUE; + retain(key); + String * ks = toString(key); + retVal = String_concat(retVal, ks); + retain(space); + retVal = String_concat(retVal, space); + retain(self->values[i]); + String * vs = toString(self->values[i]); + retVal = String_concat(retVal, vs); + } + + retVal = String_concat(retVal, closing); + release(space); + release(comma); + release(self); + return retVal; +} + +/* outside refcount system */ +void PersistentArrayMap_destroy(PersistentArrayMap *self, BOOL deallocateChildren) { + if (deallocateChildren) { + for (int i = 0; i < self->count; i++) { + void *key = self->keys[i]; + void *value = self->values[i]; + release(key); + release(value); + } + } +} + +/* outside refcount system */ +void PersistentArrayMap_retainChildren(PersistentArrayMap *self, int except) { + for (int i = 0; i < self->count; i++) { + void *foundKey = self->keys[i]; + if (foundKey && except != i) { + retain(self->keys[i]); + retain(self->values[i]); + } + } +} + +PersistentArrayMap *PersistentArrayMap_copy(PersistentArrayMap *self) { + size_t size = sizeof(PersistentArrayMap) + sizeof(Object); + Object * super = allocate(size); + PersistentArrayMap *new = Object_data(super); + retain(Object_data(self->root)); + new->root = self->root; + new->count = self->count; + Object_create(super, persistentArrayMapType); + return new; + +} + +/* mem done */ +PersistentArrayMap *PersistentArrayMap_assoc(PersistentArrayMap *self, void *key, void *value) { +// HERE in original code is some strange hasNull logic, skipping it for now +// if (key == NULL) { +// +// } + uint64_t hash = hash(key); + BOOL isNodeAdded = FALSE; + HashMapNode * root = self->root || BitmapIndexedNode_empty(); + HashMapNode * newRoot = PersistentHashMapNode_assoc(root, 0, hash, key, value, &isNodeAdded); + if (newRoot == root) { + return self; + } + + PersistentArrayMap *new = PersistentArrayMap_copy(self); + new->root = newRoot; + if (!isNodeAdded) { + new->count += 1; + } + + release(self->root); + + +} + +/* mem done */ +PersistentArrayMap *PersistentArrayMap_dissoc(PersistentArrayMap *self, void *key) { + retain(self); + int64_t found = PersistentArrayMap_indexOf(self, key); + if (found == -1) { + return self; + } + BOOL reusable = isReusable(self); + PersistentArrayMap *retVal = reusable ? self : PersistentArrayMap_copy(self); + void *k = retVal->keys[found]; + void *v = retVal->values[found]; + for (int64_t i = found; i < self->count - 1; i++) { + retVal->keys[i] = self->keys[i + 1]; + retVal->values[i] = self->values[i + 1]; + } + retVal->count--; + if (!reusable) { + PersistentArrayMap_retainChildren(retVal, -1); + release(self); + } else { + release(k); + release(v); + } + return retVal; +} + +/* mem done */ +int64_t PersistentArrayMap_indexOf(PersistentArrayMap *self, void *key) { + int64_t retVal = -1; + for (int64_t i = 0; i < self->count; i++) { + if (equals(key, self->keys[i])) { + retVal = i; + break; + } + } + release(self); + release(key); + return retVal; +} + +/* mem done */ +void *PersistentArrayMap_get(PersistentArrayMap *self, void *key) { + void *retVal = NULL; + for (int64_t i = 0; i < self->count; i++) { + if (equals(key, self->keys[i])) { + retVal = self->values[i]; + break; + } + } + if (retVal) retain(retVal); + release(self); + release(key); + return retVal ?: Nil_create(); +} + +/* mem done */ +void *PersistentArrayMap_dynamic_get(void *self, void *key) { + Object * type = super(self); + if (type->type != persistentArrayMapType) { + release(self); + release(key); + return Nil_create(); + } + return PersistentArrayMap_get(self, key); +} + +HashMapNode PersistentHashMap_createNode(uint32_t shift, void *key1, void *val1, uint32_t hash2, void *key2, + void *val2, BOOL *isNodeAdded) { + uint32_t hash1 = hash(key1); + if (hash1 == hash2) { + return HashCollisionNode_create(hash1, 2, (Object *[]) {super(key1), super(val1), super(key2), super(val2)}); + } + HashMapNode * node = BitmapIndexedNode_create(0, NULL); + + + return BitmapIndexedNode_assoc( + BitmapIndexedNode_assoc( + BitmapIndexedNode_empty(), + shift, + hash1, + key1, + val1, + isNodeAdded + ), + shift, + hash2, + key2, + val2, + isNodeAdded + ); +} + +HashCollisionNode *HashCollisionNode_create(uint32_t hash, uint32_t count, Object *array[]) { + size_t size = sizeof(Object) + sizeof(HashCollisionNode) + count * sizeof(Object *) * 2; + Object * super = allocate(size); + HashCollisionNode *self = Object_data(super); + self->hash = hash; + self->count = count; + memcpy(self->array, array, count * sizeof(Object *) * 2); + Object_create(super, hashCollisionNodeType); + return self; +} + +HashCollisionNode * +HashCollisionNode_cloneAndSet(HashCollisionNode *nodeWithArray, uint32_t idx, void *key, void *value) { + size_t newSize = sizeof(Object) + sizeof(HashCollisionNode) + nodeWithArray->count * 2 * sizeof(Object *); + Object * superValue = allocate(newSize); + HashCollisionNode *self = Object_data(superValue); + self->hash = nodeWithArray->hash; + self->count = nodeWithArray->count; + memcpy(self->array, nodeWithArray->array, nodeWithArray->count * sizeof(Object *)); + for (uint32_t i = 0; i < self->count; i++) { + if (i == idx) { + continue; + } + Object_retain(self->array[i]); + } + self->array[idx] = super(key); + self->array[idx + 1] = super(value); + Object_create(superValue, hashCollisionNodeType); + release(nodeWithArray); + return self; +} + +HashCollisionNode * +HashCollisionNode_assoc(HashCollisionNode *self, uint32_t shift, uint32_t hash, void *key, void *value, + BOOL *isNodeAdded) { + if (self->hash == hash) { + for (uint32_t i = 0; i < self->count * 2; i++) { + if (equals(self->array[i], key)) { + if (self->array[i + 1] == value) { + return self; + } + return HashCollisionNode_cloneAndSet(self, i, key, value); + } + } + *isNodeAdded = TRUE; + return HashCollisionNode_cloneAndSet(self, self->count * 2, key, value); + } + + return BitmapIndexedNode_assoc( + BitmapIndexedNode_create(BitmapIndexedNode_bitpos(self->hash, shift), (Object *[]) {NULL, self}), + shift, + hash, + key, + value, + isNodeAdded + ); +} + +BitmapIndexedNode *BitmapIndexedNode_create(uint32_t bitmap, HashMapNode *array) { + size_t size = sizeof(Object) + sizeof(BitmapIndexedNode) + __builtin_popcount(bitmap) * 2 * sizeof(Object *); + Object * super = allocate(size); + BitmapIndexedNode * self = Object_data(super); + self->bitmap = bitmap; + if (array != NULL) { + for (uint32_t i = 0; i < __builtin_popcount(bitmap) * 2; i++) { + retain(array[i]); + } + } + memcpy(self->array, array, sizeof(Object *) * __builtin_popcount(bitmap) * 2); + Object_create(super, bitmapIndexedNodeType); + return self; +} + +BitmapIndexedNode * +BitmapIndexedNode_createWithInsertion(uint32_t bitmap, uint32_t arraySize, HashMapNode *oldArray, uint32_t insertIndex, + void *keyToInsert, void *valueToInsert) { + size_t newSize = sizeof(Object) + sizeof(BitmapIndexedNode) + arraySize * sizeof(Object *); + Object * superValue = allocate(newSize); + BitmapIndexedNode * self = Object_data(superValue); + self->bitmap = bitmap; + memcpy(self->array, oldArray, sizeof(Object *) * insertIndex); + self->array[insertIndex] = super(keyToInsert); + self->array[insertIndex + 1] = super(valueToInsert); + memcpy(self->array + insertIndex + 2, oldArray + insertIndex, sizeof(Object *) * (size - insertIndex)); + Object_create(superValue, bitmapIndexedNodeType); +} + +HashMapNode BitmapIndexedNode_assoc(BitmapIndexedNode *self, uint32_t shift, uint32_t hash, void *key, void *value, + BOOL *isNodeAdded) { + + uint32_t bit = BitmapIndexedNode_bitpos(hash, shift); + uint32_t idx = BitmapIndexedNode_index(self, bit); + + Object * keySuper = super(key); + Object * valueSuper = super(value); + + if (self->bitmap & bit) { + Object * keyOrNull = self->array[2 * idx]; + Object * valueOrNode = self->array[2 * idx + 1]; + // Value is a node not a value + if (keyOrNull == NULL) { + keyOrNull = PersistentHashMapNode_assoc(valueOrNode, shift + 5, hash, key, value, isNodeAdded); + if (keyOrNull == valueOrNode) { + return self; + } + return BitmapIndexedNode_cloneAndSet(self, 2 * idx + 1, keyOrNull); + } + // Key is stored in this node + if (equals(key, keyOrNull)) { + if (value == valueOrNode) { + return self; + } + return BitmapIndexedNode_cloneAndSet(self, 2 * idx + 1, value); + } + *isNodeAdded = TRUE; + return BitmapIndexedNode_cloneAndSet( + self->array, 2 * idx, NULL, 2 * idx + 1, + PersistentHashMap_createNode( + shift + 5, keyOrNull, valueOrNode, hash, key, value, isNodeAdded)) + )); + } else { + uint8_t bitCount = __builtin_popcount(self->bitmap); + if (bitCount >= 16) { + uint32_t mask = BitmapIndexedNode_mask(hash, shift); + return ContainerNode_createFromBitmapIndexedNode(self, shift, hash, mask, key, value, isNodeAdded); + } else { + BitmapIndexedNode * result = BitmapIndexedNode_createWithInsertion(self->bitmap | bit, bitCount + 2, + self->array, idx, key, value); + *isNodeAdded = TRUE; + release(self); + return result; + } + } + +} + +ContainerNode *ContainerNode_createFromBitmapIndexedNode(BitmapIndexedNode *node, uint32_t shift, uint32_t insertHash, + uint32_t insertIndex, void *keyToInsert, void *valueToInsert, BOOL *isNodeAdded) { + uint32_t count = __builtin_popcount(node->bitmap); + size_t size = sizeof(Object) + sizeof(ContainerNode) + 32 * sizeof(Object *); // Full size node + Object * superValue = allocate(size); + ContainerNode *self = Object_data(superValue); + self->count = count + 1; + self->array[insertIndex] = BitmapIndexedNode_assoc( + BitmapIndexedNode_empty(), + shift + 5, + insertHash, + keyToInsert, + valueToInsert, + isNodeAdded + ); + uint32_t idx = 0; + for (uint32_t i = 0; i < 32; i++) { + if ((node->bitmap >> i) & 1) { + self->array[i] = node->array[j + 1]; + idx++; + } else { + self->array[i] = BitmapIndexedNode_assoc( + BitmapIndexedNode_empty(), + shift + 5, + hash(node->array[j]), + node->array[j], + node->array[j + 1], + isNodeAdded + ); + } + j += 2; + } + Object_create(super, containerNodeType); + release(node); + return self; +} + +ContainerNode *ContainerNode_cloneAndSet(ContainerNode *nodeWithArray, uint32_t idx, HashMapNode *a) { + size_t newSize = sizeof(Object) + sizeof(ContainerNode) + (nodeWithArray->count) * sizeof(Object *); + Object * super = allocate(newSize); + ContainerNode *self = Object_data(super); + self->count = self->count; + memcpy(self->array, nodeWithArray->array, (nodeWithArray->count) * + sizeof(Object *)); // unsafe cause release on nodeWithArray may cause elements of array to be released + for (uint32_t i = 0; i < self->count; i++) { + retain(self->array[i]); + } + self->array[idx] = a; + Object_create(super, containerNodeType); + release(nodeWithArray); + return self; + +} + +ContainerNode *ContainerNode_assoc(ContainerNode *self, uint32_t shift, uint32_t hash, void *key, void *value) { + uint32_t idx = BitmapIndexedNode_mask(hash, shift); + + Object * node = self->array[idx]; + + if (node == NULL) { + return ContainerNode_cloneAndSet(self, idx, key, BitmapIndexedNode_assoc( + BitmapIndexedNode_empty(), + shift + 5, + hash, + key, + value, + isNodeAdded + ) + ); + } + + HashMapNode * newNode = PersistentHashMapNode_assoc(node, shift + 5, hash, key, value, isNodeAdded); + if (newNode == node) { + return self; + } + + return ContainerNode_cloneAndSet(self, idx, newNode); + +} + +BitmapIndexedNode *ContainerNode_pack(ContainerNode *node, uint32_t index) { + uint32_t newArraySize = 2 * (node->count - 1); + size_t newSize = sizeof(Object) + sizeof(BitmapIndexedNode) + newArraySize * sizeof(Object *); + Object * super = allocate(newSize); + BitmapIndexedNode * self = Object_data(super); + self->bitmap = 0; + uint32_t j = 0; + // there are no keys here so we insert only values + for (uint32_t i = 0; i < newArraySize; i++) { + if (i == index) { + j += 2; + continue; + } + if (array[i] != NULL) { + self->bitmap |= 1 << j; + self->array[j] = node->array[i]; + j += 2; + } + } + Object_create(super, bitmapIndexedNodeType); + return self; +} + +BitmapIndexedNode *BitmapIndexedNode_createWithout(BitmapIndexedNode *node, uint32_t bitmap, uint32_t idx) { + uint32_t bitCount = __builtin_popcount(node->bitmap); + size_t newSize = sizeof(Object) + sizeof(BitmapIndexedNode) + bitCount * 2 - 2 * sizeof(Object *); + Object * super = allocate(newSize); + BitmapIndexedNode * self = Object_data(super); + self->bitmap = bitmap; + memcpy(self->array, node->array, sizeof(Object *) * idx * 2); + memcpy(self->array + idx * 2, node->array + (idx + 1) * 2, sizeof(Object *) * (self->bitmap - idx - 1) * 2); + Object_create(super, bitmapIndexedNodeType); + return self; +} + +HashMapNode BitmapIndexedNode_dissoc(BitmapIndexedNode *self, uint32_t shift, uint32_t hash, void *key) { + uint32_t bit = BitmapIndexedNode_bitpos(hash, shift); + if (!(self->bitmap & bit)) { + return self; + } + uint32_t idx = BitmapIndexedNode_index(self, bit); + Object * keyOrNull = self->array[2 * idx]; + Object * valueOrNode = self->array[2 * idx + 1]; + if (keyOrNull == NULL) { + HashMapNode * subNode = BitmapIndexedNode_dissoc(valueOrNode, shift + 5, hash, key); + if (subNode == valueOrNode) { + return self; + } + if (subNode == NULL) { + if (self->bitmap == bit) { + release(self); + return NULL; + } + return BitmapIndexedNode_createWithout(self, self->bitmap ^ bit, idx); + } + return BitmapIndexedNode_cloneAndSet(self, 2 * idx + 1, subNode); + } + if (equals(key, keyOrNull)) { + return BitmapIndexedNode_createWithout(self, self->bitmap ^ bit, idx); + } + return self; + +} + +HashMapNode *PersistentHashMapNode_assoc(HashMapNode *self, uint32_t shift, uint32_t hash, void *key, void *value, + BOOL *isNodeAdded) { + if (self == NULL) { + return NULL; + } + + switch (self->type) { + case bitmapIndexedNode: + return BitmapIndexedNode_assoc(self, shift, hash, key, value, isNodeAdded); + case hashCollisionNode: + return HashCollisionNode_assoc(self, shift, hash, key, value, isNodeAdded); + case containerNode: + return ContainerNode_assoc(self, shift, hash, key, value, isNodeAdded); + } + + assert("SHOULD NOT HAPPEN" && FALSE); + return NULL; + +} + + +Object *BitmapIndexedNode_get(BitmapIndexedNode *self, uint32_t shift, uint32_t hash, void *key); \ No newline at end of file diff --git a/backend/runtime/PersistentHashMap.h b/backend/runtime/PersistentHashMap.h new file mode 100644 index 0000000..74ed8ff --- /dev/null +++ b/backend/runtime/PersistentHashMap.h @@ -0,0 +1,149 @@ +#ifndef RT_PERSISTENT_ARRAY_MAP +#define RT_PERSISTENT_ARRAY_MAP + +#define HashMapNode Object + +#include "String.h" +#include "Object.h" + +// http://blog.higher-order.net/2009/09/08/understanding-clojures-persistenthashmap-deftwice +// https://groups.google.com/g/clojure/c/o12kFA-j1HA +// https://gist.github.com/mrange/d6e7415113ebfa52ccb660f4ce534dd4 +// https://www.infoq.com/articles/in-depth-look-clojure-collections/ + +typedef struct Object Object; +typedef struct PersistentArrayMap PersistentArrayMap; + +struct PersistentArrayMap { + uint64_t count; + HashMapNode *root; +}; + +struct BitmapIndexedNode { + uint32_t bitmap; + Object *array[]; +}; + +struct HashCollisionNode { + uint32_t hash; + uint32_t count; + Object *array[]; +}; + +struct ContainerNode { + uint32_t count; + Object *array[]; +}; + +typedef struct BitmapIndexedNode BitmapIndexedNode; +typedef struct HashCollisionNode HashCollisionNode; +typedef struct ContainerNode ContainerNode; + + +/* outside refcount system */ +inline uint32_t BitmapIndexedNode_index(BitmapIndexedNode *self, uint32_t bit) { + return __builtin_popcount(self->bitmap & (bit - 1)); +} +inline uint32_t BitmapIndexedNode_mask(uint32_t hash, uint32_t shift) { + return (hash >> shift) & 0x01f; +} +inline uint32_t BitmapIndexedNode_bitpos(uint32_t hash, uint32_t shift) { + return 1 << BitmapIndexedNode_mask(hash, shift); +} + +inline BitmapIndexedNode *BitmapIndexedNode_cloneAndSet(BitmapIndexedNode *nodeWithArray, uint32_t idx, HashMapNode* a) { + uint32_t arraySize = __builtin_popcount(nodeWithArray->bitmap); + BitmapIndexedNode *clone = allocate(sizeof(BitmapIndexedNode) * arraySize); + memcpy(clone->array, nodeWithArray->array, sizeof(BitmapIndexedNode) * arraySize); + + for (int i=0; iarray[i] == NULL || i == idx) { + continue; + } + retain(clone->array[i]); + } + + + clone->array[idx] = a; + clone->bitmap = nodeWithArray->bitmap; + + release(nodeWithArray); + return clone; +} + +inline BitmapIndexedNode *BitmapIndexedNode_cloneAndSet(BitmapIndexedNode *nodeWithArray, uint32_t idx, HashMapNode* a, uint32_t idx2, HashMapNode* a2) { + uint32_t arraySize = __builtin_popcount(nodeWithArray->bitmap); + BitmapIndexedNode *clone = allocate(sizeof(BitmapIndexedNode) * arraySize); + memcpy(clone->array, nodeWithArray->array, sizeof(BitmapIndexedNode) * arraySize); + + for (int i=0; iarray[i] == NULL || i == idx || i == idx2) { + continue; + } + retain(clone->array[i]); + } + + clone->array[idx] = a; + clone->array[idx2] = a2; + clone->bitmap = nodeWithArray->bitmap; + + release(nodeWithArray); + return clone; +} + +HashMapNode* PersistentHashMap_createNode(uint32_t shift, void *key1, void *val1, uint32_t hash2, void *key2, void *val2, BOOL *isNodeAdded); + + + +HashMapNode* BitmapIndexedNode_assoc(BitmapIndexedNode *self, uint32_t shift, uint32_t hash, void *key, void *value, BOOL *isNodeAdded); +HashMapNode* BitmapIndexedNode_dissoc(BitmapIndexedNode *self, uint32_t shift, uint32_t hash, void *key); + +BitmapIndexedNode *BitmapIndexedNode_create(uint32_t bitmap, uint32_t arraySize, HashMapNode* *array); +BitmapIndexedNode *BitmapIndexedNode_createWithout(BitmapIndexedNode *node, uint32_t bitmap, uint32_t idx); +BitmapIndexedNode *BitmapIndexedNode_createWithInsertion(uint32_t bitmap, uint32_t arraySize, HashMapNode* *oldArray, uint32_t insertIndex, void *keyToInsert, void *valueToInsert); +BitmapIndexedNode *BitmapIndexedNode_empty(); + +HashCollisionNode *HashCollisionNode_create(uint32_t hash, uint32_t count, Object *array[]); +HashCollisionNode *HashCollisionNode_cloneAndSet(HashCollisionNode *nodeWithArray, uint32_t idx, void *key, void *value); +HashCollisionNode *HashCollisionNode_assoc(HashCollisionNode *self, uint32_t shift, uint32_t hash, void *key, void *value, BOOL *isNodeAdded); +HashCollisionNode *HashCollisionNode_dissoc(HashCollisionNode *self, uint32_t hash, void *key); + +ContainerNode *ContainerNode_cloneAndSet(ContainerNode *nodeWithArray, uint32_t idx, HashMapNode* a); +BitmapIndexedNode *ContainerNode_pack(ContainerNode *node, uint32_t index); + +ContainerNode *ContainerNode_create(uint32_t count, Object *array[]); +ContainerNode *ContainerNode_createWithInsertion(ContainerNode *self, uint32_t idx, void *key, void *value); +ContainerNode *ContainerNode_createFromBitmapIndexedNode(BitmapIndexedNode *node, uint32_t shift, uint32_t insertHash, uint32_t insertIndex, void *keyToInsert, void *valueToInsert, BOOL *isNodeAdded); + +ContainerNode *ContainerNode_assoc(ContainerNode *self, uint32_t shift, uint32_t hash, void *key, void *value, BOOL *isNodeAdded); +ContainerNode *ContainerNode_dissoc(ContainerNode *self, uint32_t shift, uint32_t hash, void *key); + + +Object *BitmapIndexedNode_get(BitmapIndexedNode *self, uint32_t shift, uint32_t hash, void *key); + +HashMapNode *PersistentHashMapNode_assoc(HashMapNode *self, uint32_t shift, uint32_t hash, void *key, void *value); +HashMapNode *PersistentHashMapNode_dissoc(HashMapNode *self, uint32_t shift, uint32_t hash, void *key); +Object *PersistentHashMapNode_get(HashMapNode *self, uint32_t shift, uint32_t hash, void *key); + + + +PersistentArrayMap* PersistentArrayMap_empty(); + +BOOL PersistentArrayMap_equals(PersistentArrayMap *self, PersistentArrayMap *other); +uint64_t PersistentArrayMap_hash(PersistentArrayMap *self); +String *PersistentArrayMap_toString(PersistentArrayMap *self); +void PersistentArrayMap_destroy(PersistentArrayMap *self, BOOL deallocateChildren); + + + +PersistentArrayMap* PersistentArrayMap_assoc(PersistentArrayMap *self, void *key, void *value); +PersistentArrayMap *PersistentArrayMap_copy(PersistentArrayMap *self); +PersistentArrayMap* PersistentArrayMap_dissoc(PersistentArrayMap *self, void *key); +void* PersistentArrayMap_get(PersistentArrayMap *self, void *key); +void* PersistentArrayMap_dynamic_get(void *self, void *key); +int64_t PersistentArrayMap_indexOf(PersistentArrayMap *self, void *key); + +PersistentArrayMap* PersistentArrayMap_create(); +PersistentArrayMap* PersistentArrayMap_createMany(uint64_t pairCount, ...); + +#endif From 0cf2d9d668ec874579d9babdd8620da92408009d Mon Sep 17 00:00:00 2001 From: Mateusz Wejman Date: Mon, 13 May 2024 12:43:02 +0200 Subject: [PATCH 2/6] Problem with refCount --- backend/ObjectTypeSet.h | 64 ++- backend/RuntimeInterop.cpp | 20 +- backend/codegen.cpp | 1 + backend/runtime/BitmapIndexedNode.c | 409 ++++++++++++++++ backend/runtime/BitmapIndexedNode.h | 53 +++ backend/runtime/Boolean.h | 1 + backend/runtime/CMakeLists.txt | 47 +- backend/runtime/ContainerNode.c | 235 +++++++++ backend/runtime/ContainerNode.h | 42 ++ backend/runtime/HashCollisionNode.c | 142 ++++++ backend/runtime/HashCollisionNode.h | 36 ++ backend/runtime/Nil.c | 5 + backend/runtime/Nil.h | 1 + backend/runtime/Object.h | 150 ++++-- backend/runtime/PersistentHashMap.c | 647 +++++++------------------ backend/runtime/PersistentHashMap.h | 146 +----- backend/runtime/String.h | 1 + backend/runtime/Transient.h | 1 + backend/runtime/defines.h | 34 +- backend/runtime/runtime-tests.cpp | 707 +++++++++++++++------------- tests/assoc-persistent-hash-map.clj | 4 + 21 files changed, 1714 insertions(+), 1032 deletions(-) create mode 100644 backend/runtime/BitmapIndexedNode.c create mode 100644 backend/runtime/BitmapIndexedNode.h create mode 100644 backend/runtime/ContainerNode.c create mode 100644 backend/runtime/ContainerNode.h create mode 100644 backend/runtime/HashCollisionNode.c create mode 100644 backend/runtime/HashCollisionNode.h create mode 100644 tests/assoc-persistent-hash-map.clj diff --git a/backend/ObjectTypeSet.h b/backend/ObjectTypeSet.h index d09af28..012271f 100644 --- a/backend/ObjectTypeSet.h +++ b/backend/ObjectTypeSet.h @@ -29,7 +29,7 @@ class ConstantInteger: public ObjectTypeConstant { ConstantInteger(int64_t val) : ObjectTypeConstant(integerType), value(val) {} virtual ObjectTypeConstant *copy() { return static_cast (new ConstantInteger(value)); } virtual std::string toString() { return std::to_string(value); } - virtual bool equals(ObjectTypeConstant *other) { + virtual bool equals(ObjectTypeConstant *other) { if(ConstantInteger *i = dynamic_cast(other)) { return i->value == value; } @@ -42,7 +42,7 @@ class ConstantNil: public ObjectTypeConstant { ConstantNil() : ObjectTypeConstant(nilType) {} virtual ObjectTypeConstant *copy() { return static_cast (new ConstantNil()); } virtual std::string toString() { return "nil"; } - virtual bool equals(ObjectTypeConstant *other) { + virtual bool equals(ObjectTypeConstant *other) { return dynamic_cast(other); } }; @@ -53,7 +53,7 @@ class ConstantDouble: public ObjectTypeConstant { ConstantDouble(double val) : ObjectTypeConstant(doubleType), value(val) {} virtual ObjectTypeConstant *copy() { return static_cast (new ConstantDouble(value)); } virtual std::string toString() { return std::to_string(value); } - virtual bool equals(ObjectTypeConstant *other) { + virtual bool equals(ObjectTypeConstant *other) { if(ConstantDouble *i = dynamic_cast(other)) { return i->value == value; } @@ -67,7 +67,7 @@ class ConstantString: public ObjectTypeConstant { ConstantString(std::string val) : ObjectTypeConstant(stringType), value(val) {} virtual ObjectTypeConstant *copy() { return static_cast (new ConstantString(value)); } virtual std::string toString() { return value; } - virtual bool equals(ObjectTypeConstant *other) { + virtual bool equals(ObjectTypeConstant *other) { if(ConstantString *i = dynamic_cast(other)) { return i->value == value; } @@ -81,7 +81,7 @@ class ConstantBoolean: public ObjectTypeConstant { ConstantBoolean(bool val) : ObjectTypeConstant(booleanType), value(val) {} virtual ObjectTypeConstant *copy() { return static_cast (new ConstantBoolean(value)); } virtual std::string toString() { return std::to_string(value); } - virtual bool equals(ObjectTypeConstant *other) { + virtual bool equals(ObjectTypeConstant *other) { if(ConstantBoolean *i = dynamic_cast(other)) { return i->value == value; } @@ -109,7 +109,7 @@ class ConstantKeyword: public ObjectTypeConstant { ConstantKeyword(std::string val) : ObjectTypeConstant(keywordType), value(val) {} virtual ObjectTypeConstant *copy() { return static_cast (new ConstantKeyword(value)); } virtual std::string toString() { return value; } - virtual bool equals(ObjectTypeConstant *other) { + virtual bool equals(ObjectTypeConstant *other) { if(ConstantKeyword *i = dynamic_cast(other)) { return i->value == value; } @@ -123,7 +123,7 @@ class ConstantSymbol: public ObjectTypeConstant { ConstantSymbol(std::string val) : ObjectTypeConstant(symbolType), value(val) {} virtual ObjectTypeConstant *copy() { return static_cast (new ConstantSymbol(value)); } virtual std::string toString() { return value; } - virtual bool equals(ObjectTypeConstant *other) { + virtual bool equals(ObjectTypeConstant *other) { if(ConstantSymbol *i = dynamic_cast(other)) { return i->value == value; } @@ -137,7 +137,7 @@ class ConstantBigInteger: public ObjectTypeConstant { ConstantBigInteger(mpz_t val) : ObjectTypeConstant(bigIntegerType) { mpz_init_set(value, val); } - ConstantBigInteger(mpq_t val) : ObjectTypeConstant(bigIntegerType) { + ConstantBigInteger(mpq_t val) : ObjectTypeConstant(bigIntegerType) { assert(mpz_cmp_si(mpq_denref(val), 1) == 0); mpz_init_set(value, mpq_numref(val)); } @@ -149,7 +149,7 @@ class ConstantBigInteger: public ObjectTypeConstant { } virtual ObjectTypeConstant *copy() { return static_cast (new ConstantBigInteger(value)); } virtual std::string toString() { return std::string(mpz_get_str(NULL, 10, value)); } - virtual bool equals(ObjectTypeConstant *other) { + virtual bool equals(ObjectTypeConstant *other) { if(ConstantBigInteger *i = dynamic_cast(other)) { return mpz_cmp(i->value, value) == 0; } @@ -160,7 +160,7 @@ class ConstantBigInteger: public ObjectTypeConstant { class ConstantRatio: public ObjectTypeConstant { public: mpq_t value; - ConstantRatio(mpq_t val) : ObjectTypeConstant(ratioType) { + ConstantRatio(mpq_t val) : ObjectTypeConstant(ratioType) { mpq_init(value); mpq_set(value, val); } @@ -190,7 +190,7 @@ class ConstantRatio: public ObjectTypeConstant { } virtual ObjectTypeConstant *copy() { return static_cast (new ConstantRatio(value)); } virtual std::string toString() { return std::string(mpq_get_str(NULL, 10, value)); } - virtual bool equals(ObjectTypeConstant *other) { + virtual bool equals(ObjectTypeConstant *other) { if(ConstantRatio *i = dynamic_cast(other)) { return mpq_equal(i->value, value); } @@ -212,9 +212,9 @@ class ObjectTypeSet { ObjectTypeSet(objectType type, bool isBoxed = false, ObjectTypeConstant *cons = nullptr) : constant(cons), isBoxed(isBoxed) { internal.insert(type); - if(!constant && type == nilType) { - constant = static_cast(new ConstantNil()); - } + if(!constant && type == nilType) { + constant = static_cast(new ConstantNil()); + } if(!isScalar()) this->isBoxed = true; } ObjectTypeSet() { @@ -228,7 +228,7 @@ class ObjectTypeSet { if(other.constant) constant = other.constant->copy(); else constant = nullptr; } - + void insert(objectType type) { internal.emplace(type); } @@ -315,7 +315,7 @@ class ObjectTypeSet { if (pos == internal.end()) return; internal.erase(pos); } - + ObjectTypeSet expansion(const ObjectTypeSet &other) const { /* Expansion removes all constants */ auto retVal = ObjectTypeSet(*this); @@ -330,16 +330,16 @@ class ObjectTypeSet { /* Expansion removes all constants */ auto retVal = ObjectTypeSet(); retVal.internal = internal; - retVal.isBoxed = isBoxed; + retVal.isBoxed = isBoxed; return retVal; } - + ObjectTypeSet restriction(const ObjectTypeSet &other) const { /* Restriction preserves constant type for this */ auto retVal = ObjectTypeSet(); std::set_intersection(internal.begin(), internal.end(), - other.internal.begin(), other.internal.end(), + other.internal.begin(), other.internal.end(), std::inserter(retVal.internal, retVal.internal.begin())); retVal.isBoxed = isBoxed; if (constant != nullptr && retVal.contains(constant->constantType)) { @@ -367,7 +367,7 @@ class ObjectTypeSet { all.isBoxed = true; return all; } - + static ObjectTypeSet empty() { return ObjectTypeSet(); } @@ -386,11 +386,17 @@ class ObjectTypeSet { retVal.insert(functionType); retVal.insert(bigIntegerType); retVal.insert(ratioType); + + retVal.insert(bitmapIndexedNodeType); + retVal.insert(hashCollisionNodeType); + retVal.insert(containerNodeType); + retVal.insert(persistentHashMapType); + retVal.insert(persistentArrayMapType); retVal.isBoxed = true; return retVal; } - + static std::vector allGuesses() { auto allTypes = ObjectTypeSet::all(); std::vector guesses; @@ -403,7 +409,7 @@ class ObjectTypeSet { guesses.push_back(allTypes); return guesses; } - + static std::string typeStringForArg(const ObjectTypeSet &arg) { if(!arg.isDetermined()) return "LO"; else switch (arg.determinedType()) { @@ -435,14 +441,20 @@ class ObjectTypeSet { return "LI"; case ratioType: return "LR"; + case bitmapIndexedNodeType: + case containerNodeType: + case hashCollisionNodeType: + assert(false && "Node cannot be used as an argument"); + case persistentHashMapType: + return "LH"; case persistentArrayMapType: return "LA"; } } - + static std::string typeStringForArgs(const std::vector &args) { std::stringstream retval; - for (auto i: args) retval << typeStringForArg(i); + for (auto i: args) retval << typeStringForArg(i); return retval.str(); } @@ -452,10 +464,10 @@ class ObjectTypeSet { static std::string recursiveMethodKey(const std::string &name, const std::vector &args) { return name + "_" + typeStringForArgs(args); - } + } std::string toString() const { - std::string retVal = typeStringForArg(*this); + std::string retVal = typeStringForArg(*this); if(constant) retVal += " constant value: " + constant->toString() + " of " + std::to_string(constant->constantType); return retVal; } diff --git a/backend/RuntimeInterop.cpp b/backend/RuntimeInterop.cpp index f0fbb86..9b2d7db 100644 --- a/backend/RuntimeInterop.cpp +++ b/backend/RuntimeInterop.cpp @@ -119,6 +119,9 @@ Value *CodeGenerator::dynamicCreate(objectType type, const vector &argTy isVariadic = true; break; case persistentVectorNodeType: + case containerNodeType: + case bitmapIndexedNodeType: + case hashCollisionNodeType: throw InternalInconsistencyException("We never allow creation of subtypes here, only runtime can do it"); case nilType: fname = "Nil_create"; @@ -129,6 +132,9 @@ Value *CodeGenerator::dynamicCreate(objectType type, const vector &argTy case keywordType: fname = "Keyword_create"; break; + case persistentHashMapType: + fname = "PersistentHashMap_create"; + break; case persistentArrayMapType: fname = "PersistentArrayMap_createMany"; isVariadic = true; @@ -382,7 +388,11 @@ Value *CodeGenerator::dynamicSuper(Value *objectPtr) { // return objPtr; Value *funcPtr = Builder->CreateBitOrPointerCast(objectPtr, Type::getInt8Ty(*TheContext)->getPointerTo(), "void_to_unboxed"); // TODO: The -16 is only valid on 64 bit machines - Value *gep = Builder->CreateGEP(Type::getInt8Ty(*TheContext), funcPtr, ArrayRef((Value *)ConstantInt::get(*TheContext, APInt(64, -16)))); +#ifdef OBJECT_DEBUG + Value *gep = Builder->CreateGEP(Type::getInt8Ty(*TheContext), funcPtr, ArrayRef((Value *)ConstantInt::get(*TheContext, APInt(64, -24)))); +#else + Value *gep = Builder->CreateGEP(Type::getInt8Ty(*TheContext), funcPtr, ArrayRef((Value *)ConstantInt::get(*TheContext, APInt(64, -16)))); +#endif return Builder->CreateIntToPtr(gep, runtimeObjectType()->getPointerTo() , "sub_size"); } @@ -535,6 +545,10 @@ Type *CodeGenerator::dynamicUnboxedType(objectType type) { case symbolType: case keywordType: case persistentArrayMapType: + case persistentHashMapType: + case bitmapIndexedNodeType: + case containerNodeType: + case hashCollisionNodeType: case functionType: case concurrentHashMapType: return Type::getInt8Ty(*TheContext)->getPointerTo(); @@ -688,6 +702,10 @@ TypedValue CodeGenerator::box(const TypedValue &value) { case keywordType: case concurrentHashMapType: case persistentArrayMapType: + case persistentHashMapType: + case bitmapIndexedNodeType: + case containerNodeType: + case hashCollisionNodeType: case functionType: return TypedValue(retType, value.second); } diff --git a/backend/codegen.cpp b/backend/codegen.cpp index 04080ab..ac1e4eb 100644 --- a/backend/codegen.cpp +++ b/backend/codegen.cpp @@ -220,6 +220,7 @@ ObjectTypeSet CodeGenerator::typeForArgString(const Node &node, const string &ty if (typeName == "I") return ObjectTypeSet(bigIntegerType); if (typeName == "R") return ObjectTypeSet(ratioType); if (typeName == "A") return ObjectTypeSet(persistentArrayMapType); + if (typeName == "H") return ObjectTypeSet(persistentHashMapType); throw CodeGenerationException(string("Unknown class: ")+ typeName + string(" Full string: ") + typeString, node); } diff --git a/backend/runtime/BitmapIndexedNode.c b/backend/runtime/BitmapIndexedNode.c new file mode 100644 index 0000000..d666c6f --- /dev/null +++ b/backend/runtime/BitmapIndexedNode.c @@ -0,0 +1,409 @@ +#include "Object.h" +#include "Nil.h" +#include +#include "defines.h" +#include "PersistentHashMap.h" + +// TODO find memory corruption + +static BitmapIndexedNode *EMPTY = NULL; + +uint32_t BitmapIndexedNode_index(BitmapIndexedNode *self, uint32_t bit) { + return __builtin_popcount(self->bitmap & (bit - 1)); +} + +uint32_t BitmapIndexedNode_mask(uint32_t hash, uint32_t shift) { + return (hash >> shift) & 0x01f; +} + +uint32_t BitmapIndexedNode_bitpos(uint32_t hash, uint32_t shift) { + return 1 << BitmapIndexedNode_mask(hash, shift); +} + +BitmapIndexedNode *BitmapIndexedNode_cloneAndSet(BitmapIndexedNode *nodeWithArray, uint32_t idx, Object *a) { + #ifdef OBJECT_DEBUG + assert(a->magic == 0xdeadbeef && "Memory corruption!"); + #endif + uint32_t arraySize = __builtin_popcount(nodeWithArray->bitmap); + Object *cloneSuper = allocate(sizeof(Object) + sizeof(BitmapIndexedNode) + sizeof(Object *) * arraySize * 2); + BitmapIndexedNode *clone = Object_data(cloneSuper); + memcpy(clone->array, nodeWithArray->array, sizeof(BitmapIndexedNode) * arraySize * 2); + + for (int i = 0; i < arraySize * 2; i++) { + if (clone->array[i] == NULL || i == idx) { + continue; + } + Object_retain(clone->array[i]); + } + + + clone->array[idx] = a; + clone->bitmap = nodeWithArray->bitmap; + + Object_create(cloneSuper, bitmapIndexedNodeType); + + release(nodeWithArray); + + #ifdef OBJECT_DEBUG + assert(cloneSuper->magic == 0xdeadbeef && "Memory corruption!"); + for (int i = 0; i < arraySize * 2; i++) { + if (clone->array[i] != NULL) { + assert(clone->array[i]->magic == 0xdeadbeef && "Memory corruption!"); + } + } + #endif + + return clone; +} + +BitmapIndexedNode * +BitmapIndexedNode_cloneAndSetTwo(BitmapIndexedNode *nodeWithArray, uint32_t idx, Object *a, uint32_t idx2, Object *a2) { + #ifdef OBJECT_DEBUG + assert(a == NULL || a->magic == 0xdeadbeef && "Memory corruption!"); + assert(a2 == NULL || a2->magic == 0xdeadbeef && "Memory corruption!"); + #endif + uint32_t arraySize = __builtin_popcount(nodeWithArray->bitmap); + Object *cloneSuper = allocate(sizeof(Object) + sizeof(BitmapIndexedNode) + sizeof(Object *) * arraySize * 2); + BitmapIndexedNode *clone = Object_data(cloneSuper); + + memcpy(clone->array, nodeWithArray->array, sizeof(BitmapIndexedNode) * arraySize * 2); + + for (int i = 0; i < arraySize * 2; i++) { + if (clone->array[i] == NULL || i == idx || i == idx2) { + continue; + } + Object_retain(clone->array[i]); + } + + clone->array[idx] = a; + clone->array[idx2] = a2; + clone->bitmap = nodeWithArray->bitmap; + + Object_create(cloneSuper, bitmapIndexedNodeType); + + release(nodeWithArray); + + #ifdef OBJECT_DEBUG + assert(cloneSuper->magic == 0xdeadbeef && "Memory corruption!"); + for (int i = 0; i < arraySize * 2; i++) { + if (clone->array[i] != NULL) { + assert(clone->array[i]->magic == 0xdeadbeef && "Memory corruption!"); + } + } + #endif + + return clone; +} + +/* mem done */ +BitmapIndexedNode *BitmapIndexedNode_empty() { + if (EMPTY == NULL) EMPTY = BitmapIndexedNode_create(); + retain(EMPTY); + return EMPTY; +} + + +BitmapIndexedNode *BitmapIndexedNode_create() { + size_t size = sizeof(Object) + sizeof(BitmapIndexedNode); + Object *super = allocate(size); + BitmapIndexedNode *self = Object_data(super); + self->bitmap = 0; + Object_create(super, bitmapIndexedNodeType); + return self; +} + +BitmapIndexedNode *BitmapIndexedNode_createVa(uint32_t bitmap, uint32_t arraySize, ...) { + size_t size = sizeof(Object) + sizeof(BitmapIndexedNode) + arraySize * sizeof(Object *) * 2; + Object *superVal = allocate(size); + BitmapIndexedNode *self = Object_data(superVal); + self->bitmap = bitmap; + va_list args; + va_start(args, arraySize); + for (int i = 0; i < arraySize; i++) { + Object *key = va_arg(args, Object *); + Object *value = va_arg(args, Object *); + self->array[i * 2] = key; + self->array[i * 2 + 1] = value; + } + va_end(args); + Object_create(superVal, bitmapIndexedNodeType); + + #ifdef OBJECT_DEBUG + assert(superVal->magic == 0xdeadbeef && "Memory corruption!"); + for (int i = 0; i < arraySize * 2; i++) { + if (self->array[i] != NULL) { + assert(self->array[i]->magic == 0xdeadbeef && "Memory corruption!"); + } + } + #endif + + return self; + +} + +BitmapIndexedNode * +BitmapIndexedNode_createWithInsertion(BitmapIndexedNode *oldSelf, uint32_t arraySize, uint32_t insertIndex, + Object *keyToInsert, Object *valueToInsert, uint32_t oldBitmap) { + size_t newSize = sizeof(Object) + sizeof(BitmapIndexedNode) + sizeof(Object *) * arraySize * 2; + Object *superValue = allocate(newSize); + BitmapIndexedNode *self = Object_data(superValue); + self->bitmap = oldSelf->bitmap; + if (arraySize > 1) { + memcpy(self->array, oldSelf->array, sizeof(Object *) * insertIndex * 2); + + memcpy(self->array + 2 * insertIndex + 2, oldSelf->array + 2 * insertIndex, + sizeof(Object *) * (arraySize - insertIndex - 1) * 2); + } + + self->array[2 * insertIndex] = keyToInsert; + self->array[2 * insertIndex + 1] = valueToInsert; + + for (uint32_t i = 0; i < arraySize * 2; i++) { + if (self->array[i] == NULL || i == 2 * insertIndex || i == 2 * insertIndex + 1) { + continue; + } + Object_retain(self->array[i]); + } + + Object_create(superValue, bitmapIndexedNodeType); + + oldSelf->bitmap = oldBitmap; + release(oldSelf); + + #ifdef OBJECT_DEBUG + assert(superValue->magic == 0xdeadbeef && "Memory corruption!"); + for (int i = 0; i < arraySize * 2; i++) { + if (self->array[i] != NULL) { + assert(self->array[i]->magic == 0xdeadbeef && "Memory corruption!"); + } + } + #endif + + return self; +} + +/* mem done */ +Object *BitmapIndexedNode_assoc(BitmapIndexedNode *self, uint32_t shift, uint32_t hash, Object *key, Object *value, + BOOL *isNodeAdded) { + + #ifdef OBJECT_DEBUG + assert(key->magic == 0xdeadbeef && "Memory corruption!"); + assert(value->magic == 0xdeadbeef && "Memory corruption!"); + #endif + + uint32_t bit = BitmapIndexedNode_bitpos(hash, shift); + uint32_t idx = BitmapIndexedNode_index(self, bit); + + if (self->bitmap & bit) { + Object *keyOrNull = self->array[2 * idx]; + Object *valueOrNode = self->array[2 * idx + 1]; + + #ifdef OBJECT_DEBUG + assert(keyOrNull == NULL || keyOrNull->magic == 0xdeadbeef && "Memory corruption!"); + assert(valueOrNode || valueOrNode->magic == 0xdeadbeef && "Memory corruption!"); + #endif + + // Value is a node not a value + if (keyOrNull == NULL) { + Object_retain(valueOrNode); + keyOrNull = + PersistentHashMapNode_assoc(valueOrNode, shift + 5, hash, key, value, isNodeAdded); + + #ifdef OBJECT_DEBUG + assert(keyOrNull->magic == 0xdeadbeef && "Memory corruption!"); + #endif + + if (keyOrNull == valueOrNode) { + return super(self); + } + return super(BitmapIndexedNode_cloneAndSet(self, 2 * idx + 1, keyOrNull)); + } + // Key is stored in this node + if (Object_equals(key, keyOrNull)) { + if (Object_equals(value, valueOrNode)) { + Object_release(key); + Object_release(value); + return super(self); + } + Object_release(key); + return super(BitmapIndexedNode_cloneAndSet(self, 2 * idx + 1, value)); + } + *isNodeAdded = TRUE; + return super(BitmapIndexedNode_cloneAndSetTwo( + self, 2 * idx, NULL, 2 * idx + 1, + PersistentHashMap_createNode( + shift + 5, keyOrNull, valueOrNode, hash, key, value, isNodeAdded))); + } else { + uint8_t bitCount = __builtin_popcount(self->bitmap); + if (bitCount >= 16) { + uint32_t mask = BitmapIndexedNode_mask(hash, shift); + return ContainerNode_createFromBitmapIndexedNode(self, shift, hash, mask, key, value, isNodeAdded); + } else { + uint32_t oldBitmap = self->bitmap; + self->bitmap |= bit; + + BitmapIndexedNode *result = BitmapIndexedNode_createWithInsertion(self, bitCount + 1, idx, key, value, + oldBitmap); + #ifdef OBJECT_DEBUG + assert(super(result)->magic == 0xdeadbeef && "Memory corruption!"); + #endif + *isNodeAdded = TRUE; + return super(result); + } + } + +} + +BitmapIndexedNode *BitmapIndexedNode_createWithout(BitmapIndexedNode *node, uint32_t bitmap, uint32_t idx) { + uint32_t bitCount = __builtin_popcount(node->bitmap); + size_t newSize = sizeof(Object) + sizeof(BitmapIndexedNode) + bitCount * 2 - 2 * sizeof(Object *); //WTF + Object *super = allocate(newSize); + BitmapIndexedNode *self = Object_data(super); + self->bitmap = bitmap; + memcpy(self->array, node->array, sizeof(Object *) * idx * 2); + memcpy(self->array + idx * 2, node->array + (idx + 1) * 2, sizeof(Object *) * (self->bitmap - idx - 1) * 2); + Object_create(super, bitmapIndexedNodeType); + return self; +} + +HashMapNode *BitmapIndexedNode_dissoc(BitmapIndexedNode *self, uint32_t shift, uint32_t hash, Object *key) { +// uint32_t bit = BitmapIndexedNode_bitpos(hash, shift); +// if (!(self->bitmap & bit)) { +// return self; +// } +// uint32_t idx = BitmapIndexedNode_index(self, bit); +// Object *keyOrNull = self->array[2 * idx]; +// Object *valueOrNode = self->array[2 * idx + 1]; +// if (keyOrNull == NULL) { +// HashMapNode *subNode = BitmapIndexedNode_dissoc(valueOrNode, shift + 5, hash, key); +// if (subNode == valueOrNode) { +// return self; +// } +// if (subNode == NULL) { +// if (self->bitmap == bit) { +// release(self); +// return NULL; +// } +// return BitmapIndexedNode_createWithout(self, self->bitmap ^ bit, idx); +// } +// return BitmapIndexedNode_cloneAndSet(self, 2 * idx + 1, subNode); +// } +// if (equals(key, keyOrNull)) { +// return BitmapIndexedNode_createWithout(self, self->bitmap ^ bit, idx); +// } + return self; + +} + +Object *BitmapIndexedNode_get(BitmapIndexedNode *self, uint32_t shift, uint32_t hash, Object *key) { +// uint32_t bit = BitmapIndexedNode_bitpos(hash, shift); +// if (self->bitmap & bit) { +// uint32_t idx = BitmapIndexedNode_index(self, bit); +// Object *keyOrNull = self->array[2 * idx]; +// Object *valueOrNode = self->array[2 * idx + 1]; +// if (keyOrNull == NULL) { +// release(self); +// return PersistentHashMapNode_get(valueOrNode, shift + 5, hash, key); +// } +// if (equals(key, keyOrNull)) { +// release(self); +// return valueOrNode; +// } +// } +// release(self); + return NULL; +} + +BOOL BitmapIndexedNode_equals(BitmapIndexedNode *self, BitmapIndexedNode *other) { + if (self == other) { + return TRUE; + } + if (self->bitmap != other->bitmap) { + return FALSE; + } + for (uint32_t i = 0; i < __builtin_popcount(self->bitmap) * 2; i++) { + // CAN equals handle NULL? + Object *selfElement = self->array[i]; + Object *otherElement = other->array[i]; + if ((selfElement == NULL && otherElement != NULL) + || (selfElement != NULL && otherElement == NULL)) { + return FALSE; + } + // both are NULL (we need to check only one because of the previous if statement) + if (selfElement == NULL) { + continue; + } + if (!Object_equals(selfElement, otherElement)) { + return FALSE; + } + } + return TRUE; +} + +uint64_t BitmapIndexedNode_hash(BitmapIndexedNode *self) { + uint64_t hashValue = 5381; + for (uint32_t i = 0; i < __builtin_popcount(self->bitmap); i++) { + uint32_t keyIndex = 2 * i; + Object *key = self->array[keyIndex]; + Object *value = self->array[keyIndex + 1]; + if (value != NULL) { + if (key != NULL) { + hashValue += hash(Object_data(key)) ^ hash(Object_data(value)); + } else { + hashValue += hash(Object_data(value)); + } + } + } + return hashValue; +} + +String *BitmapIndexedNode_toString(BitmapIndexedNode *self) { +// display only when there is key and value + if (self->bitmap == 0) { + release(self); + return String_create(""); + } + String *base = String_create(""); + uint8_t idx = 0; + for (uint32_t i = 0; i < 32; i++) { + if (self->bitmap & (1 << i)) { + if (self->array[2 * idx] == NULL) { + Object_retain(self->array[2 * idx + 1]); + String *childString = toString(Object_data(self->array[2 * idx + 1])); + base = String_concat(base, childString); + + } else { + Object_retain(self->array[2 * idx]); + Object_retain(self->array[2 * idx + 1]); + String *keyString = toString(Object_data(self->array[2 * idx])); + String *valueString = toString(Object_data(self->array[2 * idx + 1])); + + base = String_concat(base, keyString); + base = String_concat(base, String_create(" ")); + base = String_concat(base, valueString); + } + if (idx < __builtin_popcount(self->bitmap) - 1) { + base = String_concat(base, String_create(", ")); + } + idx += 1; + } + } + release(self); + return base; + +} + +void BitmapIndexedNode_destroy(BitmapIndexedNode *self, BOOL deallocateChildren) { + if (deallocateChildren) { + uint8_t idx = 0; + for (uint32_t i = 0; i < __builtin_popcount(self->bitmap); i++) { + if (self->bitmap & (1 << i)) { + if (self->array[2 * idx] != NULL) { + Object_release(self->array[2 * idx]); + } + Object_release(self->array[2 * idx + 1]); + idx += 1; + } + } + } +} \ No newline at end of file diff --git a/backend/runtime/BitmapIndexedNode.h b/backend/runtime/BitmapIndexedNode.h new file mode 100644 index 0000000..0fe1d12 --- /dev/null +++ b/backend/runtime/BitmapIndexedNode.h @@ -0,0 +1,53 @@ +#ifndef RT_BITMAP_INDEXED_NODE +#define RT_BITMAP_INDEXED_NODE + +#define HashMapNode Object + +#include "String.h" +#include "Object.h" +#include +#include "ContainerNode.h" + +// http://blog.higher-order.net/2009/09/08/understanding-clojures-persistenthashmap-deftwice +// https://groups.google.com/g/clojure/c/o12kFA-j1HA +// https://gist.github.com/mrange/d6e7415113ebfa52ccb660f4ce534dd4 +// https://www.infoq.com/articles/in-depth-look-clojure-collections/ + +typedef struct Object Object; + +struct BitmapIndexedNode { + uint32_t bitmap; + Object *array[]; +}; + +typedef struct BitmapIndexedNode BitmapIndexedNode; + +uint32_t BitmapIndexedNode_index(BitmapIndexedNode *self, uint32_t bit); +uint32_t BitmapIndexedNode_mask(uint32_t hash, uint32_t shift); +uint32_t BitmapIndexedNode_bitpos(uint32_t hash, uint32_t shift); + +BitmapIndexedNode *BitmapIndexedNode_cloneAndSet(BitmapIndexedNode *nodeWithArray, uint32_t idx, HashMapNode* a); + +BitmapIndexedNode *BitmapIndexedNode_cloneAndSetTwo(BitmapIndexedNode *nodeWithArray, uint32_t idx, Object * a, uint32_t idx2, Object * a2); + + + +Object* BitmapIndexedNode_assoc(BitmapIndexedNode *self, uint32_t shift, uint32_t hash, Object *key, Object *value, BOOL *isNodeAdded); +HashMapNode* BitmapIndexedNode_dissoc(BitmapIndexedNode *self, uint32_t shift, uint32_t hash, Object *key); + +BitmapIndexedNode *BitmapIndexedNode_create(); +BitmapIndexedNode *BitmapIndexedNode_createVa(uint32_t bitmap, uint32_t arraySize, ...); +BitmapIndexedNode *BitmapIndexedNode_createWithout(BitmapIndexedNode *node, uint32_t bitmap, uint32_t idx); +BitmapIndexedNode *BitmapIndexedNode_createWithInsertion(BitmapIndexedNode *oldSelf, uint32_t arraySize, uint32_t insertIndex, Object *keyToInsert, Object *valueToInsert, uint32_t oldBitmap); +BitmapIndexedNode *BitmapIndexedNode_empty(); + + +Object *BitmapIndexedNode_get(BitmapIndexedNode *self, uint32_t shift, uint32_t hash, Object *key); + +BOOL BitmapIndexedNode_equals(BitmapIndexedNode *self, BitmapIndexedNode *other); +uint64_t BitmapIndexedNode_hash(BitmapIndexedNode *self); +String *BitmapIndexedNode_toString(BitmapIndexedNode *self); +void BitmapIndexedNode_destroy(BitmapIndexedNode *self, BOOL deallocateChildren); + + +#endif diff --git a/backend/runtime/Boolean.h b/backend/runtime/Boolean.h index 53f1a57..0bf9535 100644 --- a/backend/runtime/Boolean.h +++ b/backend/runtime/Boolean.h @@ -3,6 +3,7 @@ #include "String.h" #include "defines.h" + struct Boolean { unsigned char value; }; diff --git a/backend/runtime/CMakeLists.txt b/backend/runtime/CMakeLists.txt index ddf173d..f2f6add 100644 --- a/backend/runtime/CMakeLists.txt +++ b/backend/runtime/CMakeLists.txt @@ -3,9 +3,9 @@ project(runtime) #set(CMAKE_OSX_ARCHITECTURES "arm64") -if(NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE Release) -endif() +if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif () #set(CMAKE_CXX_FLAGS "-Wall -Wextra") set(CMAKE_CXX_FLAGS_DEBUG "-g") @@ -16,24 +16,29 @@ set(CMAKE_C_FLAGS_DEBUG "-g") set(CMAKE_C_FLAGS_RELEASE "-Ofast") set(SOURCES - Transient.c - Object.c - Integer.c - PersistentList.c - PersistentVector.c - PersistentVectorNode.c - String.c - Double.c - Nil.c - Boolean.c - Symbol.c - ConcurrentHashMap.c - Interface.c - Keyword.c - Function.c - BigInteger.c - Ratio.c - PersistentArrayMap.c) + Transient.c + Object.c + Integer.c + PersistentList.c + PersistentVector.c + PersistentVectorNode.c + String.c + Double.c + Nil.c + Boolean.c + Symbol.c + ConcurrentHashMap.c + Interface.c + Keyword.c + Function.c + BigInteger.c + Ratio.c + PersistentArrayMap.c + BitmapIndexedNode.c + ContainerNode.c + HashCollisionNode.c + PersistentHashMap.c +) add_library(runtime STATIC ${SOURCES}) diff --git a/backend/runtime/ContainerNode.c b/backend/runtime/ContainerNode.c new file mode 100644 index 0000000..ec34cec --- /dev/null +++ b/backend/runtime/ContainerNode.c @@ -0,0 +1,235 @@ +#include "Object.h" +#include "ContainerNode.h" +#include "Nil.h" +#include "defines.h" +#include "PersistentHashMap.h" + +Object *ContainerNode_createFromBitmapIndexedNode(BitmapIndexedNode *node, uint32_t shift, uint32_t insertHash, + uint32_t insertIndex, Object *keyToInsert, Object *valueToInsert, + BOOL *isNodeAdded) { + uint32_t count = __builtin_popcount(node->bitmap); + size_t size = sizeof(Object) + sizeof(ContainerNode) + 32 * sizeof(Object *); // Full size node + Object *superValue = allocate(size); + ContainerNode *self = Object_data(superValue); + self->count = count + 1; + self->array[insertIndex] = BitmapIndexedNode_assoc( + BitmapIndexedNode_empty(), + shift + 5, + insertHash, + keyToInsert, + valueToInsert, + isNodeAdded + ); + + #ifdef OBJECT_DEBUG + assert(keyToInsert->magic == 0xdeadbeef && "Memory corruption!"); + assert(valueToInsert->magic == 0xdeadbeef && "Memory corruption!"); + #endif + uint32_t idx = 0; + // NAPRAWIĆ TO, COŚ TU NIE DZIAŁA! + for (uint32_t i = 0; i < 32; i++) { + if ((node->bitmap >> i) & 1) { + // we have a key, value pair here + if (node->array[2 * idx] != NULL) { + Object_retain(node->array[2 * idx]); + Object_retain(node->array[2 * idx + 1]); + self->array[i] = BitmapIndexedNode_assoc( + BitmapIndexedNode_empty(), + shift + 5, + hash(Object_data(node->array[2 * idx])), + node->array[2 * idx], + node->array[2 * idx + 1], + isNodeAdded + ); + } else if (node->array[2 * idx + 1] != NULL) { + Object_retain(node->array[2 * idx + 1]); + self->array[i] = node->array[2 * idx + 1]; + } + idx += 1; + } else { + self->array[i] = super(BitmapIndexedNode_empty()); + } + } + + Object_create(superValue, containerNodeType); + release(node); + + #ifdef OBJECT_DEBUG + assert(superValue->magic == 0xdeadbeef && "Memory corruption!"); + for (uint32_t i = 0; i < 32; i++) { + if (self->array[i] != NULL) { + assert(self->array[i]->magic == 0xdeadbeef && "Memory corruption!"); + } + } + #endif + + return superValue; +} + +ContainerNode *ContainerNode_cloneAndSet(ContainerNode *nodeWithArray, uint32_t idx, Object *a) { + + #ifdef OBJECT_DEBUG + assert(a->magic == 0xdeadbeef && "Memory corruption!"); + #endif + + size_t newSize = sizeof(Object) + sizeof(ContainerNode) + 32 * sizeof(Object *); + Object *superValue = allocate(newSize); + ContainerNode *self = Object_data(superValue); + self->count = nodeWithArray->count; + memcpy(self->array, nodeWithArray->array, 32 * + sizeof(Object *)); + for (uint32_t i = 0; i < 32; i++) { + if (i == idx || self->array[i] == NULL) { + continue; + } + Object_retain(self->array[i]); + } + self->array[idx] = a; + Object_create(superValue, containerNodeType); + release(nodeWithArray); + + #ifdef OBJECT_DEBUG + assert(superValue->magic == 0xdeadbeef && "Memory corruption!"); + for (uint32_t i = 0; i < 32; i++) { + if (self->array[i] != NULL) { + assert(self->array[i]->magic == 0xdeadbeef && "Memory corruption!"); + } + } + #endif + + return self; + +} + +Object * +ContainerNode_assoc(ContainerNode *self, uint32_t shift, uint32_t hash, Object *key, Object *value, BOOL *isNodeAdded) { + uint32_t idx = BitmapIndexedNode_mask(hash, shift); + + if (idx == 2) { + printf("Breakpoint here\n"); + } + + Object *node = self->array[idx]; + + #ifdef OBJECT_DEBUG + assert(key->magic == 0xdeadbeef && "Memory corruption!"); + assert(value->magic == 0xdeadbeef && "Memory corruption!"); + assert(node == NULL || node->magic == 0xdeadbeef && "Memory corruption!"); + #endif + + + if (node == NULL) { + return super(ContainerNode_cloneAndSet(self, idx, BitmapIndexedNode_assoc( + BitmapIndexedNode_empty(), + shift + 5, + hash, + key, + value, + isNodeAdded + )) + ); + } + + Object *newNode = PersistentHashMapNode_assoc(node, shift + 5, hash, key, value, isNodeAdded); + #ifdef OBJECT_DEBUG + assert(newNode->magic == 0xdeadbeef && "Memory corruption!"); + #endif + if (newNode == node) { + Object_release(newNode); // Maybe? + return super(self); + } + + #ifdef OBJECT_DEBUG + assert(newNode->magic == 0xdeadbeef && "Memory corruption!"); + ContainerNode *result = ContainerNode_cloneAndSet(self, idx, newNode); + assert(super(result)->magic == 0xdeadbeef && "Memory corruption!"); + return super(result); + #else + return super(ContainerNode_cloneAndSet(self, idx, newNode)); + #endif + +} + +BitmapIndexedNode *ContainerNode_pack(ContainerNode *node, uint32_t index) { + uint32_t newArraySize = 2 * (node->count - 1); + size_t newSize = sizeof(Object) + sizeof(BitmapIndexedNode) + newArraySize * sizeof(Object *); + Object *super = allocate(newSize); + BitmapIndexedNode *self = Object_data(super); + self->bitmap = 0; + uint32_t j = 0; + // there are no keys here so we insert only values + for (uint32_t i = 0; i < newArraySize; i++) { + if (i == index) { + j += 2; + continue; + } + if (self->array[i] != NULL) { + self->bitmap |= 1 << j; + self->array[j] = node->array[i]; + j += 2; + } + } + Object_create(super, bitmapIndexedNodeType); + return self; +} + +BOOL ContainerNode_equals(ContainerNode *self, ContainerNode *other) { + if (self->count != other->count) { + return FALSE; + } + for (uint32_t i = 0; i < self->count; i++) { + if (self->array[i] == NULL) { + if (other->array[i] != NULL) { + return FALSE; + } + i -= 1; + } else { + if (!Object_equals(self->array[i], other->array[i])) { + return FALSE; + } + } + } + return TRUE; +} + +uint64_t ContainerNode_hash(ContainerNode *self) { + uint64_t hashValue = 0; + for (uint32_t i = 0; i < self->count; i++) { + if (self->array[i] != NULL) { + hashValue += hash(Object_data(self->array[i])); + } + } + return hashValue; +} + +String *ContainerNode_toString(ContainerNode *self) { + String *base = String_create(""); + uint8_t checked = 0; + for (uint32_t i = 0; i < 32; i++) { + if (self->array[i] != NULL) { + Object_retain(self->array[i]); + String *childRepr = toString(Object_data(self->array[i])); + retain(childRepr); + base = String_concat(base, childRepr); + if (checked <= self->count && childRepr->count > 0) { + base = String_concat(base, String_create(", ")); + } + release(childRepr); + if (childRepr->count > 0) { + checked += 1; + } + } + } + release(self); + return base; +} + +void ContainerNode_destroy(ContainerNode *self, BOOL deallocateChildren) { + if (deallocateChildren) { + for (uint32_t i = 0; i < 32; i++) { + if (self->array[i] != NULL) { + Object_release(self->array[i]); + } + } + } +} \ No newline at end of file diff --git a/backend/runtime/ContainerNode.h b/backend/runtime/ContainerNode.h new file mode 100644 index 0000000..66b8de8 --- /dev/null +++ b/backend/runtime/ContainerNode.h @@ -0,0 +1,42 @@ +#ifndef RT_CONTAINER_NODE +#define RT_CONTAINER_NODE + +#define HashMapNode Object + +#include "String.h" +#include "Object.h" +#include "BitmapIndexedNode.h" +#include + +// http://blog.higher-order.net/2009/09/08/understanding-clojures-persistenthashmap-deftwice +// https://groups.google.com/g/clojure/c/o12kFA-j1HA +// https://gist.github.com/mrange/d6e7415113ebfa52ccb660f4ce534dd4 +// https://www.infoq.com/articles/in-depth-look-clojure-collections/ + +typedef struct Object Object; + +struct ContainerNode { + uint32_t count; + Object *array[]; +}; + +typedef struct ContainerNode ContainerNode; +typedef struct BitmapIndexedNode BitmapIndexedNode; + +ContainerNode *ContainerNode_cloneAndSet(ContainerNode *nodeWithArray, uint32_t idx, HashMapNode* a); +BitmapIndexedNode *ContainerNode_pack(ContainerNode *node, uint32_t index); + +ContainerNode *ContainerNode_create(uint32_t count, Object *array[]); +ContainerNode *ContainerNode_createWithInsertion(ContainerNode *self, uint32_t idx, void *key, void *value); +Object *ContainerNode_createFromBitmapIndexedNode(BitmapIndexedNode *node, uint32_t shift, uint32_t insertHash, uint32_t insertIndex, Object *keyToInsert, Object *valueToInsert, BOOL *isNodeAdded); + +Object *ContainerNode_assoc(ContainerNode *self, uint32_t shift, uint32_t hash, Object *key, Object *value, BOOL *isNodeAdded); +ContainerNode *ContainerNode_dissoc(ContainerNode *self, uint32_t shift, uint32_t hash, void *key); + + +BOOL ContainerNode_equals(ContainerNode *self, ContainerNode *other); +uint64_t ContainerNode_hash(ContainerNode *self); +String *ContainerNode_toString(ContainerNode *self); +void ContainerNode_destroy(ContainerNode *self, BOOL deallocateChildren); + +#endif diff --git a/backend/runtime/HashCollisionNode.c b/backend/runtime/HashCollisionNode.c new file mode 100644 index 0000000..e5221d2 --- /dev/null +++ b/backend/runtime/HashCollisionNode.c @@ -0,0 +1,142 @@ +#include "Object.h" +#include "HashCollisionNode.h" +#include "Nil.h" +#include +#include "defines.h" + + +HashCollisionNode *HashCollisionNode_create(uint32_t hash, uint32_t count, ...) { + size_t size = sizeof(Object) + sizeof(HashCollisionNode) + count * sizeof(Object *) * 2; + Object * superVal = allocate(size); + HashCollisionNode *self = Object_data(superVal); + self->hash = hash; + self->count = count; + va_list args; + va_start(args, count); + for (int i = 0; i < count; i++) { + Object *key = va_arg(args, Object *); + Object *value = va_arg(args, Object *); + self->array[i * 2] = key; + self->array[i * 2 + 1] = value; + } + va_end(args); + Object_create(superVal, hashCollisionNodeType); + + #ifdef OBJECT_DEBUG + assert(superVal->magic == 0xdeadbeef && "Memory corruption!"); + for (uint32_t i = 0; i < 32; i++) { + if (self->array[i] != NULL) { + assert(self->array[i]->magic == 0xdeadbeef && "Memory corruption!"); + } + } + #endif + + return self; +} + +HashCollisionNode * +HashCollisionNode_cloneAndSet(HashCollisionNode *nodeWithArray, uint32_t idx, Object *key, Object *value) { + #ifdef OBJECT_DEBUG + assert(key->magic == 0xdeadbeef && "Memory corruption!"); + assert(value->magic == 0xdeadbeef && "Memory corruption!"); + + #endif + size_t newSize = sizeof(Object) + sizeof(HashCollisionNode) + nodeWithArray->count * 2 * sizeof(Object *); + Object * superValue = allocate(newSize); + HashCollisionNode *self = Object_data(superValue); + self->hash = nodeWithArray->hash; + self->count = nodeWithArray->count; + memcpy(self->array, nodeWithArray->array, nodeWithArray->count * sizeof(Object *)); + for (uint32_t i = 0; i < self->count; i++) { + if (i == idx || i == idx + 1) { + continue; + } + Object_retain(self->array[i]); + } + self->array[idx] = key; + self->array[idx + 1] = value; + Object_create(superValue, hashCollisionNodeType); + release(nodeWithArray); + + #ifdef OBJECT_DEBUG + assert(superValue->magic == 0xdeadbeef && "Memory corruption!"); + for (uint32_t i = 0; i < 32; i++) { + if (self->array[i] != NULL) { + assert(self->array[i]->magic == 0xdeadbeef && "Memory corruption!"); + } + } + #endif + + return self; +} + +Object * +HashCollisionNode_assoc(HashCollisionNode *self, uint32_t shift, uint32_t hash, Object *key, Object *value, + BOOL *isNodeAdded) { + #ifdef OBJECT_DEBUG + assert(key->magic == 0xdeadbeef && "Memory corruption!"); + assert(value->magic == 0xdeadbeef && "Memory corruption!"); + #endif + if (self->hash == hash) { + for (uint32_t i = 0; i < self->count; i++) { + if (Object_equals(self->array[2 * i], key)) { + if (Object_equals(self->array[2 * i + 1], value)) { + return super(self); + } + return super(HashCollisionNode_cloneAndSet(self, 2 * i, key, value)); + } + } + *isNodeAdded = TRUE; + return super(HashCollisionNode_cloneAndSet(self, self->count * 2, key, value)); + } + + return BitmapIndexedNode_assoc( + BitmapIndexedNode_createVa(BitmapIndexedNode_bitpos(self->hash, shift), 1, NULL, super(self)), + shift, + hash, + key, + value, + isNodeAdded + ); +} + +void HashCollisionNode_destroy(HashCollisionNode *self, BOOL deallocateChildren) { + if (deallocateChildren) { + for (uint32_t i = 0; i < self->count * 2; i++) { + Object_release(self->array[i]); + } + } +} +uint64_t HashCollisionNode_hash(HashCollisionNode *self) { + uint64_t hashValue = 0; + for (uint32_t i = 0; i < self->count; i++) { + hashValue += hash(Object_data(self->array[2 * i])) ^ hash(Object_data(self->array[2 * i + 1])); + } + return hashValue; +} +BOOL HashCollisionNode_equals(HashCollisionNode *self, HashCollisionNode *other) { + if (self->count != other->count) { + return FALSE; + } + for (uint32_t i = 0; i < self->count * 2; i++) { + if (!Object_equals(self->array[i], other->array[i])) { + return FALSE; + } + } + return TRUE; +} +String *HashCollisionNode_toString(HashCollisionNode *self) { + String *base = String_create(""); + for (uint32_t i = 0; i < self->count; i++) { + Object_retain(self->array[i * 2]); + Object_retain(self->array[i * 2 + 1]); + base = String_concat(base, toString(Object_data(self->array[i * 2]))); + base = String_concat(base, String_create(" ")); + base = String_concat(base, toString(Object_data(self->array[i * 2 + 1]))); + if (i != self->count - 1) { + base = String_concat(base, String_create(", ")); + } + } + release(self); + return base; +} \ No newline at end of file diff --git a/backend/runtime/HashCollisionNode.h b/backend/runtime/HashCollisionNode.h new file mode 100644 index 0000000..8983dc6 --- /dev/null +++ b/backend/runtime/HashCollisionNode.h @@ -0,0 +1,36 @@ +#ifndef RT_HASH_COLLISION_NODE +#define RT_HASH_COLLISION_NODE + +#define HashMapNode Object + +#include "String.h" +#include "Object.h" +#include + +// http://blog.higher-order.net/2009/09/08/understanding-clojures-persistenthashmap-deftwice +// https://groups.google.com/g/clojure/c/o12kFA-j1HA +// https://gist.github.com/mrange/d6e7415113ebfa52ccb660f4ce534dd4 +// https://www.infoq.com/articles/in-depth-look-clojure-collections/ + +typedef struct Object Object; + +struct HashCollisionNode { + uint32_t hash; + uint32_t count; + Object *array[]; +}; + +typedef struct HashCollisionNode HashCollisionNode; + + +HashCollisionNode *HashCollisionNode_create(uint32_t hash, uint32_t count, ...); +HashCollisionNode *HashCollisionNode_cloneAndSet(HashCollisionNode *nodeWithArray, uint32_t idx, Object *key, Object *value); +Object *HashCollisionNode_assoc(HashCollisionNode *self, uint32_t shift, uint32_t hash, Object *key, Object *value, BOOL *isNodeAdded); +HashCollisionNode *HashCollisionNode_dissoc(HashCollisionNode *self, uint32_t hash, Object *key); + +void HashCollisionNode_destroy(HashCollisionNode *self, BOOL deallocateChildren); +uint64_t HashCollisionNode_hash(HashCollisionNode *self); +BOOL HashCollisionNode_equals(HashCollisionNode *self, HashCollisionNode *other); +String *HashCollisionNode_toString(HashCollisionNode *self); + +#endif diff --git a/backend/runtime/Nil.c b/backend/runtime/Nil.c index bfbc528..3dcbfb6 100644 --- a/backend/runtime/Nil.c +++ b/backend/runtime/Nil.c @@ -39,6 +39,11 @@ String *Nil_toString(Nil *self) { return String_create("nil"); } +/* outside refcount system */ +Nil * const Nil_getUnique() { + return UNIQUE_NIL; +} + /* outside refcount system */ void Nil_destroy(Nil *self) { } diff --git a/backend/runtime/Nil.h b/backend/runtime/Nil.h index bc3b1cc..de9d538 100644 --- a/backend/runtime/Nil.h +++ b/backend/runtime/Nil.h @@ -13,6 +13,7 @@ BOOL Nil_equals(Nil *self, Nil *other); uint64_t Nil_hash(Nil *self); String *Nil_toString(Nil *self); void Nil_destroy(Nil *self); +Nil *const Nil_getUnique(); void Nil_initialise(); diff --git a/backend/runtime/Object.h b/backend/runtime/Object.h index 1176744..de21481 100644 --- a/backend/runtime/Object.h +++ b/backend/runtime/Object.h @@ -1,10 +1,12 @@ #ifndef RT_OBJECT #define RT_OBJECT +#define OBJECT_DEBUG 1 #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wnullability-completeness" #pragma clang diagnostic ignored "-Wexpansion-to-defined" +#include #include #include #include @@ -28,14 +30,21 @@ #include "Function.h" #include "BigInteger.h" #include "Ratio.h" +#include "PersistentHashMap.h" #include "PersistentArrayMap.h" +#include "BitmapIndexedNode.h" +#include "HashCollisionNode.h" +#include "ContainerNode.h" -typedef struct String String; +typedef struct String String; extern void logBacktrace(); void printReferenceCounts(); struct Object { +#ifdef OBJECT_DEBUG + uint64_t magic; +#endif objectType type; #ifdef REFCOUNT_NONATOMIC uint64_t refCount; @@ -44,9 +53,9 @@ struct Object { #endif }; -typedef struct Object Object; +typedef struct Object Object; -extern _Atomic uint64_t allocationCount[256]; +extern _Atomic uint64_t allocationCount[256]; void initialise_memory(); @@ -56,7 +65,7 @@ inline void *allocate(size_t size) { /* if (size <= 128) return poolMalloc(&globalPool2); */ /* if (size > 64 && size <= BLOCK_SIZE) return poolMalloc(&globalPool1); */ assert(size > 0 && "Size = 0"); - void * retVal = malloc(size); + void * retVal = malloc(size); assert(retVal && "Could not aloocate that much! "); return retVal; } @@ -65,42 +74,55 @@ inline void deallocate(void * restrict ptr) { /* if (poolFreeCheck(ptr, &globalPool3)) { poolFree(&globalPool3, ptr); return; } */ /* if (poolFreeCheck(ptr, &globalPool2)) { poolFree(&globalPool2, ptr); return; } */ /* if (poolFreeCheck(ptr, &globalPool1)) { poolFree(&globalPool1, ptr); return; } */ - free(ptr); + free(ptr); } inline void *Object_data(Object * restrict self) { +//#ifdef OBJECT_DEBUG +// assert(self->magic == 0xdeadbeef && "Memory corruption!"); +//#endif return self + 1; } -inline Object *super(void * restrict self) { - return (Object *)(self - sizeof(Object)); +inline Object *super(void * const restrict self) { +#ifdef OBJECT_DEBUG + Object * obj = (Object *)(self - sizeof(Object)); + assert(obj->magic == 0xdeadbeef && "Memory corruption!"); +#endif + return (Object *)(self - sizeof(Object)); } inline void Object_retain(Object * restrict self) { +#ifdef OBJECT_DEBUG + assert(self->magic == 0xdeadbeef && "Memory corruption!"); +#endif // printf("RETAIN!!! %d\n", self->type); #ifdef REFCOUNT_TRACING atomic_fetch_add_explicit(&(allocationCount[self->type-1]), 1, memory_order_relaxed); #endif #ifdef REFCOUNT_NONATOMIC self->refCount++; -#else +#else atomic_fetch_add_explicit(&(self->atomicRefCount), 1, memory_order_relaxed); #endif } inline void Object_destroy(Object *restrict self, BOOL deallocateChildren) { +#ifdef OBJECT_DEBUG + assert(self->magic == 0xdeadbeef && "Memory corruption!"); +#endif // printf("--> Deallocating type %d addres %lld\n", self->type, (uint64_t)Object_data(self)); // printReferenceCounts(); switch((objectType)self->type) { case integerType: Integer_destroy(Object_data(self)); - break; + break; case stringType: String_destroy(Object_data(self)); - break; + break; case persistentListType: PersistentList_destroy(Object_data(self), deallocateChildren); - break; + break; case persistentVectorType: PersistentVector_destroy(Object_data(self), deallocateChildren); break; @@ -109,7 +131,7 @@ inline void Object_destroy(Object *restrict self, BOOL deallocateChildren) { break; case doubleType: Double_destroy(Object_data(self)); - break; + break; case booleanType: Boolean_destroy(Object_data(self)); break; @@ -134,6 +156,18 @@ inline void Object_destroy(Object *restrict self, BOOL deallocateChildren) { case ratioType: Ratio_destroy(Object_data(self)); break; + case bitmapIndexedNodeType: + BitmapIndexedNode_destroy(Object_data(self), deallocateChildren); + break; + case hashCollisionNodeType: + HashCollisionNode_destroy(Object_data(self), deallocateChildren); + break; + case containerNodeType: + ContainerNode_destroy(Object_data(self), deallocateChildren); + break; + case persistentHashMapType: + PersistentHashMap_destroy(Object_data(self), deallocateChildren); + break; case persistentArrayMapType: PersistentArrayMap_destroy(Object_data(self), deallocateChildren); break; @@ -146,11 +180,11 @@ inline void Object_destroy(Object *restrict self, BOOL deallocateChildren) { inline BOOL Object_isReusable(Object *restrict self) { uint64_t refCount = atomic_load_explicit(&(self->atomicRefCount), memory_order_relaxed); - // Multithreading - is it really safe to assume it is reusable if refcount is 1? - /* The reasoning here is that passing object to another thread is an operation that by - itself increases its reference count. Therefore it is assumed that if recount is 1 at - a point in time this means other threads have no knowledge of this object's existence - at this particular moment. Therefore, if only our thread knows of it, it can pass it to another + // Multithreading - is it really safe to assume it is reusable if refcount is 1? + /* The reasoning here is that passing object to another thread is an operation that by + itself increases its reference count. Therefore it is assumed that if recount is 1 at + a point in time this means other threads have no knowledge of this object's existence + at this particular moment. Therefore, if only our thread knows of it, it can pass it to another but only after it completes current operation. */ return refCount == 1; @@ -161,6 +195,9 @@ inline BOOL isReusable(void *restrict self) { } inline BOOL Object_release_internal(Object * restrict self, BOOL deallocateChildren) { +#ifdef OBJECT_DEBUG + assert(self->magic == 0xdeadbeef && "Memory corruption!"); +#endif #ifdef REFCOUNT_TRACING // printf("RELEASE!!! %p %d %d\n", self, self->type, deallocateChildren); atomic_fetch_sub_explicit(&(allocationCount[self->type -1 ]), 1, memory_order_relaxed); @@ -169,7 +206,7 @@ inline BOOL Object_release_internal(Object * restrict self, BOOL deallocateChild #ifdef REFCOUNT_NONATOMIC if (--self->refCount == 0) { #else - uint64_t relVal = atomic_fetch_sub_explicit(&(self->atomicRefCount), 1, memory_order_relaxed); + uint64_t relVal = atomic_fetch_sub_explicit(&(self->atomicRefCount), 1, memory_order_relaxed); assert(relVal >= 1 && "Memory corruption!"); if (relVal == 1) { #endif @@ -180,31 +217,26 @@ inline BOOL Object_release_internal(Object * restrict self, BOOL deallocateChild } inline uint64_t Object_hash(Object * restrict self) { +#ifdef OBJECT_DEBUG + assert(self->magic == 0xdeadbeef && "Memory corruption!"); +#endif switch((objectType)self->type) { case integerType: return Integer_hash(Object_data(self)); - break; case stringType: return String_hash(Object_data(self)); - break; case persistentListType: return PersistentList_hash(Object_data(self)); - break; case persistentVectorType: return PersistentVector_hash(Object_data(self)); - break; case persistentVectorNodeType: return PersistentVector_hash(Object_data(self)); - break; case doubleType: return Double_hash(Object_data(self)); - break; case booleanType: return Boolean_hash(Object_data(self)); - break; case nilType: return Nil_hash(Object_data(self)); - break; case symbolType: return Symbol_hash(Object_data(self)); case concurrentHashMapType: @@ -217,8 +249,16 @@ inline uint64_t Object_hash(Object * restrict self) { return BigInteger_hash(Object_data(self)); case ratioType: return Ratio_hash(Object_data(self)); + case bitmapIndexedNodeType: + return BitmapIndexedNode_hash(Object_data(self)); + case hashCollisionNodeType: + return HashCollisionNode_hash(Object_data(self)); + case containerNodeType: + return ContainerNode_hash(Object_data(self)); + case persistentHashMapType: + return PersistentHashMap_hash(Object_data(self)); case persistentArrayMapType: - return PersistentArrayMap_hash(Object_data(self)); + return PersistentArrayMap_hash(Object_data(self)); } } @@ -226,24 +266,28 @@ inline uint64_t hash(void * restrict self) { return Object_hash(super(self)); } -inline BOOL Object_equals(Object * restrict self, Object * restrict other) { +inline BOOL Object_equals(Object * const restrict self, Object * const restrict other) { +#ifdef OBJECT_DEBUG + assert(self->magic == 0xdeadbeef && "Memory corruption!"); + assert(other->magic == 0xdeadbeef && "Memory corruption!"); +#endif if (self == other) return TRUE; if (self->type != other->type) return FALSE; if (Object_hash(self) != Object_hash(other)) return FALSE; void *selfData = Object_data(self); - void *otherData = Object_data(other); + void *otherData = Object_data(other); switch((objectType)self->type) { case integerType: return Integer_equals(selfData, otherData); - break; + break; case stringType: return String_equals(selfData, otherData); - break; + break; case persistentListType: return PersistentList_equals(selfData, otherData); - break; + break; case persistentVectorType: return PersistentVector_equals(selfData, otherData); break; @@ -277,28 +321,39 @@ inline BOOL Object_equals(Object * restrict self, Object * restrict other) { case ratioType: return Ratio_equals(selfData, otherData); break; + case bitmapIndexedNodeType: + return BitmapIndexedNode_equals(selfData, otherData); + case hashCollisionNodeType: + return HashCollisionNode_equals(selfData, otherData); + case containerNodeType: + return ContainerNode_equals(selfData, otherData); + case persistentHashMapType: + return PersistentHashMap_equals(selfData, otherData); case persistentArrayMapType: return PersistentArrayMap_equals(selfData, otherData); break; } } -inline BOOL equals(void * restrict self, void * restrict other) { +inline BOOL equals(void * const restrict self, void * const restrict other) { return Object_equals(super(self), super(other)); } inline String *Object_toString(Object * restrict self) { +#ifdef OBJECT_DEBUG + assert(self->magic == 0xdeadbeef && "Memory corruption!"); +#endif switch((objectType)self->type) { case integerType: return Integer_toString(Object_data(self)); - break; + break; case stringType: return String_toString(Object_data(self)); - break; + break; case persistentListType: return PersistentList_toString(Object_data(self)); - break; + break; case persistentVectorType: return PersistentVector_toString(Object_data(self)); break; @@ -326,6 +381,14 @@ inline String *Object_toString(Object * restrict self) { return BigInteger_toString(Object_data(self)); case ratioType: return Ratio_toString(Object_data(self)); + case bitmapIndexedNodeType: + return BitmapIndexedNode_toString(Object_data(self)); + case hashCollisionNodeType: + return HashCollisionNode_toString(Object_data(self)); + case containerNodeType: + return ContainerNode_toString(Object_data(self)); + case persistentHashMapType: + return PersistentHashMap_toString(Object_data(self)); case persistentArrayMapType: return PersistentArrayMap_toString(Object_data(self)); } @@ -336,6 +399,9 @@ inline String *toString(void * restrict self) { } inline BOOL Object_release(Object * restrict self) { +#ifdef OBJECT_DEBUG + assert(self->magic == 0xdeadbeef && "Memory corruption!"); +#endif return Object_release_internal(self, TRUE); } @@ -344,11 +410,14 @@ inline objectType getType(void *obj) { } inline void Object_autorelease(Object * restrict self) { +#ifdef OBJECT_DEBUG + assert(self->magic == 0xdeadbeef && "Memory corruption!"); +#endif /* The object could have been deallocated through direct releases in the meantime (e.g. if autoreleasing entity does not own ) */ #ifdef REFCOUNT_NONATOMIC - if(self->refCount < 1) return; + if(self->refCount < 1) return; #else - if(atomic_load(&(self->atomicRefCount)) < 1) return; + if(atomic_load(&(self->atomicRefCount)) < 1) return; #endif /* TODO: add an object to autorelease pool */ /* If we have no other threads working, we release immediately */ @@ -368,6 +437,11 @@ inline BOOL release(void * restrict self) { } inline void Object_create(Object * restrict self, objectType type) { + +#ifdef OBJECT_DEBUG + self->magic = 0xdeadbeef; +#endif + #ifdef REFCOUNT_NONATOMIC self->refCount = 1; #else diff --git a/backend/runtime/PersistentHashMap.c b/backend/runtime/PersistentHashMap.c index 335c8ee..cb6c06a 100644 --- a/backend/runtime/PersistentHashMap.c +++ b/backend/runtime/PersistentHashMap.c @@ -1,568 +1,247 @@ #include "Object.h" -#include "PersistentArrayMap.h" +#include "PersistentHashMap.h" #include "Nil.h" #include #include "defines.h" -static PersistentArrayMap *EMPTY = NULL; + +static PersistentHashMap *EMPTY = NULL; /* mem done */ -PersistentArrayMap *PersistentArrayMap_empty() { - if (EMPTY == NULL) EMPTY = PersistentArrayMap_create(); +PersistentHashMap *PersistentHashMap_empty() { + if (EMPTY == NULL) EMPTY = PersistentHashMap_create(); retain(EMPTY); return EMPTY; } /* mem done */ -PersistentArrayMap *PersistentArrayMap_create() { - size_t size = sizeof(Object) + sizeof(PersistentArrayMap); - Object * super = allocate(size); - PersistentArrayMap *self = Object_data(super); - memset(super, 0, size); - Object_create(super, persistentArrayMapType); - return self; -} - -/* outside refcount system */ -PersistentArrayMap *PersistentArrayMap_copy(PersistentArrayMap *other) { - size_t baseSize = sizeof(PersistentArrayMap); - size_t size = baseSize + sizeof(Object); - Object * super = allocate(size); - PersistentArrayMap *self = Object_data(super); - memcpy(self, other, baseSize); - Object_create(super, persistentArrayMapType); +PersistentHashMap *PersistentHashMap_create() { + size_t size = sizeof(Object) + sizeof(PersistentHashMap); + Object *superVal = allocate(size); + PersistentHashMap *self = Object_data(superVal); + memset(superVal, 0, size); + Object_create(superVal, persistentHashMapType); return self; } /* mem done */ -PersistentArrayMap *PersistentArrayMap_createMany(uint64_t pairCount, ...) { - assert(pairCount < HASHTABLE_THRESHOLD + 1 && "Maps of size > 8 not supported yet"); - va_list args; - va_start(args, pairCount); - - PersistentArrayMap *self = PersistentArrayMap_create(); - - self->count = pairCount; - for (int i = 0; i < pairCount; i++) { - void *k = va_arg(args, - void *); - void *v = va_arg(args, - void *); - self->keys[i] = k; - self->values[i] = v; - } - va_end(args); - return self; -} +//PersistentHashMap *PersistentHashMap_createMany(uint64_t pairCount, ...) { +// assert(pairCount < HASHTABLE_THRESHOLD + 1 && "Maps of size > 8 not supported yet"); +// va_list args; +// va_start(args, pairCount); +// +// PersistentHashMap *self = PersistentHashMap_create(); +// +// self->count = pairCount; +// for (int i = 0; i < pairCount; i++) { +// void *k = va_arg(args, +// void *); +// void *v = va_arg(args, +// void *); +// self->keys[i] = k; +// self->values[i] = v; +// } +// va_end(args); +// return self; +//} /* outside refcount system */ -BOOL PersistentArrayMap_equals(PersistentArrayMap *self, PersistentArrayMap *other) { +BOOL PersistentHashMap_equals(PersistentHashMap *const self, PersistentHashMap *const other) { if (self->count != other->count) return FALSE; - for (int i = 0; i < self->count; i++) { - void *key = self->keys[i]; - void *val = self->values[i]; - retain(self); - retain(key); - void *otherVal = PersistentArrayMap_get(other, key); - if (!equals(otherVal, val)) return FALSE; + + if (self->root == NULL && other->root == NULL) { + return TRUE; } - return TRUE; -} -/* outside refcount system */ -uint64_t PersistentArrayMap_hash(PersistentArrayMap *self) { - uint64_t h = 5381; - for (int i = 0; i < self->count; i++) { - void *key = self->keys[i]; - void *value = self->values[i]; - h += (hash(key) ^ hash(value)); + if (self->nilValue != other->nilValue) { + return FALSE; } - return h; -} -/* mem done */ -String *PersistentArrayMap_toString(PersistentArrayMap *self) { - String * retVal = String_create("{"); - String * space = String_create(" "); - String * comma = String_create(", "); - String * closing = String_create("}"); - - BOOL hasAtLeastOne = FALSE; - for (int i = 0; i < self->count; i++) { - void *key = self->keys[i]; - if (hasAtLeastOne) { - retain(comma); - retVal = String_concat(retVal, comma); - } - hasAtLeastOne = TRUE; - retain(key); - String * ks = toString(key); - retVal = String_concat(retVal, ks); - retain(space); - retVal = String_concat(retVal, space); - retain(self->values[i]); - String * vs = toString(self->values[i]); - retVal = String_concat(retVal, vs); + if (self->root == NULL || other->root == NULL) { + return FALSE; } - retVal = String_concat(retVal, closing); - release(space); - release(comma); - release(self); - return retVal; + return equals(self->root, other->root); } /* outside refcount system */ -void PersistentArrayMap_destroy(PersistentArrayMap *self, BOOL deallocateChildren) { - if (deallocateChildren) { - for (int i = 0; i < self->count; i++) { - void *key = self->keys[i]; - void *value = self->values[i]; - release(key); - release(value); - } - } -} +uint64_t PersistentHashMap_hash(PersistentHashMap *const self) { + uint64_t h = 5381 + hash(Object_data(self->root)); -/* outside refcount system */ -void PersistentArrayMap_retainChildren(PersistentArrayMap *self, int except) { - for (int i = 0; i < self->count; i++) { - void *foundKey = self->keys[i]; - if (foundKey && except != i) { - retain(self->keys[i]); - retain(self->values[i]); - } + if (self->nilValue != NULL) { + Nil *nil = Nil_create(); + h += hash(Object_data(self->nilValue)); + h += hash(nil); + release(nil); } -} - -PersistentArrayMap *PersistentArrayMap_copy(PersistentArrayMap *self) { - size_t size = sizeof(PersistentArrayMap) + sizeof(Object); - Object * super = allocate(size); - PersistentArrayMap *new = Object_data(super); - retain(Object_data(self->root)); - new->root = self->root; - new->count = self->count; - Object_create(super, persistentArrayMapType); - return new; + return h; } /* mem done */ -PersistentArrayMap *PersistentArrayMap_assoc(PersistentArrayMap *self, void *key, void *value) { -// HERE in original code is some strange hasNull logic, skipping it for now -// if (key == NULL) { -// -// } - uint64_t hash = hash(key); - BOOL isNodeAdded = FALSE; - HashMapNode * root = self->root || BitmapIndexedNode_empty(); - HashMapNode * newRoot = PersistentHashMapNode_assoc(root, 0, hash, key, value, &isNodeAdded); - if (newRoot == root) { - return self; - } +String *PersistentHashMap_toString(PersistentHashMap *const self) { + String *retVal = String_create("{"); + String *closing = String_create("}"); - PersistentArrayMap *new = PersistentArrayMap_copy(self); - new->root = newRoot; - if (!isNodeAdded) { - new->count += 1; + if (self->nilValue != NULL) { + Object_retain(self->nilValue); + String *nilStr = toString(Object_data(self->nilValue)); + retVal = String_concat(retVal, String_create("nil ")); + retVal = String_concat(retVal, nilStr); + retVal = String_concat(retVal, String_create(", ")); } - release(self->root); - - -} - -/* mem done */ -PersistentArrayMap *PersistentArrayMap_dissoc(PersistentArrayMap *self, void *key) { - retain(self); - int64_t found = PersistentArrayMap_indexOf(self, key); - if (found == -1) { - return self; - } - BOOL reusable = isReusable(self); - PersistentArrayMap *retVal = reusable ? self : PersistentArrayMap_copy(self); - void *k = retVal->keys[found]; - void *v = retVal->values[found]; - for (int64_t i = found; i < self->count - 1; i++) { - retVal->keys[i] = self->keys[i + 1]; - retVal->values[i] = self->values[i + 1]; - } - retVal->count--; - if (!reusable) { - PersistentArrayMap_retainChildren(retVal, -1); + if (self->root == NULL) { release(self); - } else { - release(k); - release(v); + return String_concat(retVal, closing); } - return retVal; -} -/* mem done */ -int64_t PersistentArrayMap_indexOf(PersistentArrayMap *self, void *key) { - int64_t retVal = -1; - for (int64_t i = 0; i < self->count; i++) { - if (equals(key, self->keys[i])) { - retVal = i; - break; - } - } - release(self); - release(key); - return retVal; -} + Object_retain(self->root); + String *rootStr = toString(Object_data(self->root)); -/* mem done */ -void *PersistentArrayMap_get(PersistentArrayMap *self, void *key) { - void *retVal = NULL; - for (int64_t i = 0; i < self->count; i++) { - if (equals(key, self->keys[i])) { - retVal = self->values[i]; - break; - } - } - if (retVal) retain(retVal); + retVal = String_concat(retVal, rootStr); release(self); - release(key); - return retVal ?: Nil_create(); + return String_concat(retVal, closing); } -/* mem done */ -void *PersistentArrayMap_dynamic_get(void *self, void *key) { - Object * type = super(self); - if (type->type != persistentArrayMapType) { - release(self); - release(key); - return Nil_create(); +/* outside refcount system */ +void PersistentHashMap_destroy(PersistentHashMap *self, BOOL deallocateChildren) { + if (self->root != NULL) { + Object_destroy(self->root, deallocateChildren); } - return PersistentArrayMap_get(self, key); -} - -HashMapNode PersistentHashMap_createNode(uint32_t shift, void *key1, void *val1, uint32_t hash2, void *key2, - void *val2, BOOL *isNodeAdded) { - uint32_t hash1 = hash(key1); - if (hash1 == hash2) { - return HashCollisionNode_create(hash1, 2, (Object *[]) {super(key1), super(val1), super(key2), super(val2)}); + if (self->nilValue != NULL) { + Object_destroy(self->nilValue, deallocateChildren); } - HashMapNode * node = BitmapIndexedNode_create(0, NULL); - - - return BitmapIndexedNode_assoc( - BitmapIndexedNode_assoc( - BitmapIndexedNode_empty(), - shift, - hash1, - key1, - val1, - isNodeAdded - ), - shift, - hash2, - key2, - val2, - isNodeAdded - ); } -HashCollisionNode *HashCollisionNode_create(uint32_t hash, uint32_t count, Object *array[]) { - size_t size = sizeof(Object) + sizeof(HashCollisionNode) + count * sizeof(Object *) * 2; - Object * super = allocate(size); - HashCollisionNode *self = Object_data(super); - self->hash = hash; - self->count = count; - memcpy(self->array, array, count * sizeof(Object *) * 2); - Object_create(super, hashCollisionNodeType); - return self; -} +/* outside refcount system */ +//void PersistentHashMap_retainChildren(PersistentHashMap *self, int except) { +// for (int i = 0; i < self->count; i++) { +// void *foundKey = self->keys[i]; +// if (foundKey && except != i) { +// retain(self->keys[i]); +// retain(self->values[i]); +// } +// } +//} -HashCollisionNode * -HashCollisionNode_cloneAndSet(HashCollisionNode *nodeWithArray, uint32_t idx, void *key, void *value) { - size_t newSize = sizeof(Object) + sizeof(HashCollisionNode) + nodeWithArray->count * 2 * sizeof(Object *); - Object * superValue = allocate(newSize); - HashCollisionNode *self = Object_data(superValue); - self->hash = nodeWithArray->hash; - self->count = nodeWithArray->count; - memcpy(self->array, nodeWithArray->array, nodeWithArray->count * sizeof(Object *)); - for (uint32_t i = 0; i < self->count; i++) { - if (i == idx) { - continue; - } - Object_retain(self->array[i]); +PersistentHashMap *PersistentHashMap_copy(PersistentHashMap *self) { + size_t size = sizeof(Object) + sizeof(PersistentHashMap); + Object *superVal = allocate(size); + PersistentHashMap *new = Object_data(superVal); + if (self->root != NULL) { + Object_retain(self->root); } - self->array[idx] = super(key); - self->array[idx + 1] = super(value); - Object_create(superValue, hashCollisionNodeType); - release(nodeWithArray); - return self; -} -HashCollisionNode * -HashCollisionNode_assoc(HashCollisionNode *self, uint32_t shift, uint32_t hash, void *key, void *value, - BOOL *isNodeAdded) { - if (self->hash == hash) { - for (uint32_t i = 0; i < self->count * 2; i++) { - if (equals(self->array[i], key)) { - if (self->array[i + 1] == value) { - return self; - } - return HashCollisionNode_cloneAndSet(self, i, key, value); - } - } - *isNodeAdded = TRUE; - return HashCollisionNode_cloneAndSet(self, self->count * 2, key, value); + if (self->nilValue != NULL) { + Object_retain(self->nilValue); + new->nilValue = self->nilValue; + } else { + new->nilValue = NULL; } - return BitmapIndexedNode_assoc( - BitmapIndexedNode_create(BitmapIndexedNode_bitpos(self->hash, shift), (Object *[]) {NULL, self}), - shift, - hash, - key, - value, - isNodeAdded - ); -} + new->root = self->root; + new->count = self->count; -BitmapIndexedNode *BitmapIndexedNode_create(uint32_t bitmap, HashMapNode *array) { - size_t size = sizeof(Object) + sizeof(BitmapIndexedNode) + __builtin_popcount(bitmap) * 2 * sizeof(Object *); - Object * super = allocate(size); - BitmapIndexedNode * self = Object_data(super); - self->bitmap = bitmap; - if (array != NULL) { - for (uint32_t i = 0; i < __builtin_popcount(bitmap) * 2; i++) { - retain(array[i]); - } - } - memcpy(self->array, array, sizeof(Object *) * __builtin_popcount(bitmap) * 2); - Object_create(super, bitmapIndexedNodeType); - return self; -} + Object_create(superVal, persistentHashMapType); + release(self); + return new; -BitmapIndexedNode * -BitmapIndexedNode_createWithInsertion(uint32_t bitmap, uint32_t arraySize, HashMapNode *oldArray, uint32_t insertIndex, - void *keyToInsert, void *valueToInsert) { - size_t newSize = sizeof(Object) + sizeof(BitmapIndexedNode) + arraySize * sizeof(Object *); - Object * superValue = allocate(newSize); - BitmapIndexedNode * self = Object_data(superValue); - self->bitmap = bitmap; - memcpy(self->array, oldArray, sizeof(Object *) * insertIndex); - self->array[insertIndex] = super(keyToInsert); - self->array[insertIndex + 1] = super(valueToInsert); - memcpy(self->array + insertIndex + 2, oldArray + insertIndex, sizeof(Object *) * (size - insertIndex)); - Object_create(superValue, bitmapIndexedNodeType); } -HashMapNode BitmapIndexedNode_assoc(BitmapIndexedNode *self, uint32_t shift, uint32_t hash, void *key, void *value, - BOOL *isNodeAdded) { - - uint32_t bit = BitmapIndexedNode_bitpos(hash, shift); - uint32_t idx = BitmapIndexedNode_index(self, bit); - - Object * keySuper = super(key); - Object * valueSuper = super(value); - - if (self->bitmap & bit) { - Object * keyOrNull = self->array[2 * idx]; - Object * valueOrNode = self->array[2 * idx + 1]; - // Value is a node not a value - if (keyOrNull == NULL) { - keyOrNull = PersistentHashMapNode_assoc(valueOrNode, shift + 5, hash, key, value, isNodeAdded); - if (keyOrNull == valueOrNode) { - return self; - } - return BitmapIndexedNode_cloneAndSet(self, 2 * idx + 1, keyOrNull); - } - // Key is stored in this node - if (equals(key, keyOrNull)) { - if (value == valueOrNode) { - return self; - } - return BitmapIndexedNode_cloneAndSet(self, 2 * idx + 1, value); - } - *isNodeAdded = TRUE; - return BitmapIndexedNode_cloneAndSet( - self->array, 2 * idx, NULL, 2 * idx + 1, - PersistentHashMap_createNode( - shift + 5, keyOrNull, valueOrNode, hash, key, value, isNodeAdded)) - )); - } else { - uint8_t bitCount = __builtin_popcount(self->bitmap); - if (bitCount >= 16) { - uint32_t mask = BitmapIndexedNode_mask(hash, shift); - return ContainerNode_createFromBitmapIndexedNode(self, shift, hash, mask, key, value, isNodeAdded); - } else { - BitmapIndexedNode * result = BitmapIndexedNode_createWithInsertion(self->bitmap | bit, bitCount + 2, - self->array, idx, key, value); - *isNodeAdded = TRUE; - release(self); - return result; +/* mem done */ +PersistentHashMap *PersistentHashMap_assoc(PersistentHashMap *const self, Object *const key, Object *const value) { + if (equals(Object_data(key), Nil_getUnique())) { + if (equals(Object_data(value), Object_data(self->nilValue))) { + Object_release(key); + Object_release(value); + return self; } + PersistentHashMap *new = PersistentHashMap_copy(self); + new->nilValue = value; + new->count += 1; + Object_release(key); + return new; } + uint64_t hashValue = Object_hash(key); + BOOL isNodeAdded = FALSE; + Object *root = self->root; -} + if (root == NULL) { + root = super(BitmapIndexedNode_empty()); + } -ContainerNode *ContainerNode_createFromBitmapIndexedNode(BitmapIndexedNode *node, uint32_t shift, uint32_t insertHash, - uint32_t insertIndex, void *keyToInsert, void *valueToInsert, BOOL *isNodeAdded) { - uint32_t count = __builtin_popcount(node->bitmap); - size_t size = sizeof(Object) + sizeof(ContainerNode) + 32 * sizeof(Object *); // Full size node - Object * superValue = allocate(size); - ContainerNode *self = Object_data(superValue); - self->count = count + 1; - self->array[insertIndex] = BitmapIndexedNode_assoc( - BitmapIndexedNode_empty(), - shift + 5, - insertHash, - keyToInsert, - valueToInsert, - isNodeAdded + Object_retain(root); + Object *newRoot = PersistentHashMapNode_assoc( + root, + 0, + hashValue, + key, + value, + &isNodeAdded ); - uint32_t idx = 0; - for (uint32_t i = 0; i < 32; i++) { - if ((node->bitmap >> i) & 1) { - self->array[i] = node->array[j + 1]; - idx++; - } else { - self->array[i] = BitmapIndexedNode_assoc( - BitmapIndexedNode_empty(), - shift + 5, - hash(node->array[j]), - node->array[j], - node->array[j + 1], - isNodeAdded - ); - } - j += 2; + if (newRoot == root) { + Object_release(root); + return self; } - Object_create(super, containerNodeType); - release(node); - return self; -} -ContainerNode *ContainerNode_cloneAndSet(ContainerNode *nodeWithArray, uint32_t idx, HashMapNode *a) { - size_t newSize = sizeof(Object) + sizeof(ContainerNode) + (nodeWithArray->count) * sizeof(Object *); - Object * super = allocate(newSize); - ContainerNode *self = Object_data(super); - self->count = self->count; - memcpy(self->array, nodeWithArray->array, (nodeWithArray->count) * - sizeof(Object *)); // unsafe cause release on nodeWithArray may cause elements of array to be released - for (uint32_t i = 0; i < self->count; i++) { - retain(self->array[i]); + PersistentHashMap *new = PersistentHashMap_copy(self); + new->root = newRoot; + if (isNodeAdded) { + new->count += 1; } - self->array[idx] = a; - Object_create(super, containerNodeType); - release(nodeWithArray); - return self; - + return new; } -ContainerNode *ContainerNode_assoc(ContainerNode *self, uint32_t shift, uint32_t hash, void *key, void *value) { - uint32_t idx = BitmapIndexedNode_mask(hash, shift); - - Object * node = self->array[idx]; - - if (node == NULL) { - return ContainerNode_cloneAndSet(self, idx, key, BitmapIndexedNode_assoc( - BitmapIndexedNode_empty(), - shift + 5, - hash, - key, - value, - isNodeAdded - ) - ); - } +Object *PersistentHashMapNode_assoc(Object *const self, uint32_t shift, uint32_t hash, Object *const key, Object *const value, + BOOL *isNodeAdded) { +// NULL should not be a case here for self +#ifdef OBJECT_DEBUG + assert(self->magic == 0xdeadbeef && "Memory corruption!"); + assert(key->magic == 0xdeadbeef && "Memory corruption!"); + assert(value->magic == 0xdeadbeef && "Memory corruption!"); +#endif - HashMapNode * newNode = PersistentHashMapNode_assoc(node, shift + 5, hash, key, value, isNodeAdded); - if (newNode == node) { - return self; + switch (self->type) { + case bitmapIndexedNodeType: + return BitmapIndexedNode_assoc(Object_data(self), shift, hash, key, value, isNodeAdded); + case containerNodeType: + return ContainerNode_assoc(Object_data(self), shift, hash, key, value, isNodeAdded); + case hashCollisionNodeType: + return HashCollisionNode_assoc(Object_data(self), shift, hash, key, value, isNodeAdded); + default: + assert(FALSE && "Should not happen for PersistentHashMapNode_assoc"); } - - return ContainerNode_cloneAndSet(self, idx, newNode); - } -BitmapIndexedNode *ContainerNode_pack(ContainerNode *node, uint32_t index) { - uint32_t newArraySize = 2 * (node->count - 1); - size_t newSize = sizeof(Object) + sizeof(BitmapIndexedNode) + newArraySize * sizeof(Object *); - Object * super = allocate(newSize); - BitmapIndexedNode * self = Object_data(super); - self->bitmap = 0; - uint32_t j = 0; - // there are no keys here so we insert only values - for (uint32_t i = 0; i < newArraySize; i++) { - if (i == index) { - j += 2; - continue; - } - if (array[i] != NULL) { - self->bitmap |= 1 << j; - self->array[j] = node->array[i]; - j += 2; - } - } - Object_create(super, bitmapIndexedNodeType); - return self; +PersistentHashMap *PersistentHashMap_dissoc(PersistentHashMap *self, void *key) { + return NULL; } -BitmapIndexedNode *BitmapIndexedNode_createWithout(BitmapIndexedNode *node, uint32_t bitmap, uint32_t idx) { - uint32_t bitCount = __builtin_popcount(node->bitmap); - size_t newSize = sizeof(Object) + sizeof(BitmapIndexedNode) + bitCount * 2 - 2 * sizeof(Object *); - Object * super = allocate(newSize); - BitmapIndexedNode * self = Object_data(super); - self->bitmap = bitmap; - memcpy(self->array, node->array, sizeof(Object *) * idx * 2); - memcpy(self->array + idx * 2, node->array + (idx + 1) * 2, sizeof(Object *) * (self->bitmap - idx - 1) * 2); - Object_create(super, bitmapIndexedNodeType); - return self; +int64_t PersistentHashMap_indexOf(PersistentHashMap *self, void *key) { + return -1; } -HashMapNode BitmapIndexedNode_dissoc(BitmapIndexedNode *self, uint32_t shift, uint32_t hash, void *key) { - uint32_t bit = BitmapIndexedNode_bitpos(hash, shift); - if (!(self->bitmap & bit)) { - return self; - } - uint32_t idx = BitmapIndexedNode_index(self, bit); - Object * keyOrNull = self->array[2 * idx]; - Object * valueOrNode = self->array[2 * idx + 1]; - if (keyOrNull == NULL) { - HashMapNode * subNode = BitmapIndexedNode_dissoc(valueOrNode, shift + 5, hash, key); - if (subNode == valueOrNode) { - return self; - } - if (subNode == NULL) { - if (self->bitmap == bit) { - release(self); - return NULL; - } - return BitmapIndexedNode_createWithout(self, self->bitmap ^ bit, idx); - } - return BitmapIndexedNode_cloneAndSet(self, 2 * idx + 1, subNode); - } - if (equals(key, keyOrNull)) { - return BitmapIndexedNode_createWithout(self, self->bitmap ^ bit, idx); - } - return self; +void *PersistentHashMap_get(PersistentHashMap *self, void *key) { + return NULL; +} +void *PersistentHashMap_dynamic_get(void *self, void *key) { + return NULL; } -HashMapNode *PersistentHashMapNode_assoc(HashMapNode *self, uint32_t shift, uint32_t hash, void *key, void *value, - BOOL *isNodeAdded) { - if (self == NULL) { - return NULL; +Object *PersistentHashMap_createNode(uint32_t shift, Object *const key1, Object *const val1, uint32_t hash2, Object *const key2, + Object *const val2, BOOL *isNodeAdded) { + uint32_t hash1 = Object_hash(key1); + if (hash1 == hash2) { + return super(HashCollisionNode_create(hash1, 2, key1, val1, key2, val2)); } - switch (self->type) { - case bitmapIndexedNode: - return BitmapIndexedNode_assoc(self, shift, hash, key, value, isNodeAdded); - case hashCollisionNode: - return HashCollisionNode_assoc(self, shift, hash, key, value, isNodeAdded); - case containerNode: - return ContainerNode_assoc(self, shift, hash, key, value, isNodeAdded); - } - assert("SHOULD NOT HAPPEN" && FALSE); - return NULL; + BitmapIndexedNode *node = BitmapIndexedNode_empty(); -} + node = Object_data(BitmapIndexedNode_assoc(node, shift, hash1, key1, val1, isNodeAdded)); + return BitmapIndexedNode_assoc(node, shift, hash2, key2, val2, isNodeAdded); +} -Object *BitmapIndexedNode_get(BitmapIndexedNode *self, uint32_t shift, uint32_t hash, void *key); \ No newline at end of file diff --git a/backend/runtime/PersistentHashMap.h b/backend/runtime/PersistentHashMap.h index 74ed8ff..aa93aed 100644 --- a/backend/runtime/PersistentHashMap.h +++ b/backend/runtime/PersistentHashMap.h @@ -1,8 +1,9 @@ -#ifndef RT_PERSISTENT_ARRAY_MAP -#define RT_PERSISTENT_ARRAY_MAP +#ifndef RT_PERSISTENT_HASH_MAP +#define RT_PERSISTENT_HASH_MAP #define HashMapNode Object +#include #include "String.h" #include "Object.h" @@ -12,138 +13,39 @@ // https://www.infoq.com/articles/in-depth-look-clojure-collections/ typedef struct Object Object; -typedef struct PersistentArrayMap PersistentArrayMap; +typedef struct PersistentHashMap PersistentHashMap; -struct PersistentArrayMap { +struct PersistentHashMap { uint64_t count; - HashMapNode *root; + Object *root; + Object *nilValue; }; -struct BitmapIndexedNode { - uint32_t bitmap; - Object *array[]; -}; - -struct HashCollisionNode { - uint32_t hash; - uint32_t count; - Object *array[]; -}; - -struct ContainerNode { - uint32_t count; - Object *array[]; -}; - -typedef struct BitmapIndexedNode BitmapIndexedNode; -typedef struct HashCollisionNode HashCollisionNode; -typedef struct ContainerNode ContainerNode; - - -/* outside refcount system */ -inline uint32_t BitmapIndexedNode_index(BitmapIndexedNode *self, uint32_t bit) { - return __builtin_popcount(self->bitmap & (bit - 1)); -} -inline uint32_t BitmapIndexedNode_mask(uint32_t hash, uint32_t shift) { - return (hash >> shift) & 0x01f; -} -inline uint32_t BitmapIndexedNode_bitpos(uint32_t hash, uint32_t shift) { - return 1 << BitmapIndexedNode_mask(hash, shift); -} - -inline BitmapIndexedNode *BitmapIndexedNode_cloneAndSet(BitmapIndexedNode *nodeWithArray, uint32_t idx, HashMapNode* a) { - uint32_t arraySize = __builtin_popcount(nodeWithArray->bitmap); - BitmapIndexedNode *clone = allocate(sizeof(BitmapIndexedNode) * arraySize); - memcpy(clone->array, nodeWithArray->array, sizeof(BitmapIndexedNode) * arraySize); - - for (int i=0; iarray[i] == NULL || i == idx) { - continue; - } - retain(clone->array[i]); - } - - - clone->array[idx] = a; - clone->bitmap = nodeWithArray->bitmap; - - release(nodeWithArray); - return clone; -} - -inline BitmapIndexedNode *BitmapIndexedNode_cloneAndSet(BitmapIndexedNode *nodeWithArray, uint32_t idx, HashMapNode* a, uint32_t idx2, HashMapNode* a2) { - uint32_t arraySize = __builtin_popcount(nodeWithArray->bitmap); - BitmapIndexedNode *clone = allocate(sizeof(BitmapIndexedNode) * arraySize); - memcpy(clone->array, nodeWithArray->array, sizeof(BitmapIndexedNode) * arraySize); - - for (int i=0; iarray[i] == NULL || i == idx || i == idx2) { - continue; - } - retain(clone->array[i]); - } - - clone->array[idx] = a; - clone->array[idx2] = a2; - clone->bitmap = nodeWithArray->bitmap; - - release(nodeWithArray); - return clone; -} - -HashMapNode* PersistentHashMap_createNode(uint32_t shift, void *key1, void *val1, uint32_t hash2, void *key2, void *val2, BOOL *isNodeAdded); - - - -HashMapNode* BitmapIndexedNode_assoc(BitmapIndexedNode *self, uint32_t shift, uint32_t hash, void *key, void *value, BOOL *isNodeAdded); -HashMapNode* BitmapIndexedNode_dissoc(BitmapIndexedNode *self, uint32_t shift, uint32_t hash, void *key); - -BitmapIndexedNode *BitmapIndexedNode_create(uint32_t bitmap, uint32_t arraySize, HashMapNode* *array); -BitmapIndexedNode *BitmapIndexedNode_createWithout(BitmapIndexedNode *node, uint32_t bitmap, uint32_t idx); -BitmapIndexedNode *BitmapIndexedNode_createWithInsertion(uint32_t bitmap, uint32_t arraySize, HashMapNode* *oldArray, uint32_t insertIndex, void *keyToInsert, void *valueToInsert); -BitmapIndexedNode *BitmapIndexedNode_empty(); - -HashCollisionNode *HashCollisionNode_create(uint32_t hash, uint32_t count, Object *array[]); -HashCollisionNode *HashCollisionNode_cloneAndSet(HashCollisionNode *nodeWithArray, uint32_t idx, void *key, void *value); -HashCollisionNode *HashCollisionNode_assoc(HashCollisionNode *self, uint32_t shift, uint32_t hash, void *key, void *value, BOOL *isNodeAdded); -HashCollisionNode *HashCollisionNode_dissoc(HashCollisionNode *self, uint32_t hash, void *key); - -ContainerNode *ContainerNode_cloneAndSet(ContainerNode *nodeWithArray, uint32_t idx, HashMapNode* a); -BitmapIndexedNode *ContainerNode_pack(ContainerNode *node, uint32_t index); - -ContainerNode *ContainerNode_create(uint32_t count, Object *array[]); -ContainerNode *ContainerNode_createWithInsertion(ContainerNode *self, uint32_t idx, void *key, void *value); -ContainerNode *ContainerNode_createFromBitmapIndexedNode(BitmapIndexedNode *node, uint32_t shift, uint32_t insertHash, uint32_t insertIndex, void *keyToInsert, void *valueToInsert, BOOL *isNodeAdded); - -ContainerNode *ContainerNode_assoc(ContainerNode *self, uint32_t shift, uint32_t hash, void *key, void *value, BOOL *isNodeAdded); -ContainerNode *ContainerNode_dissoc(ContainerNode *self, uint32_t shift, uint32_t hash, void *key); - - -Object *BitmapIndexedNode_get(BitmapIndexedNode *self, uint32_t shift, uint32_t hash, void *key); +Object* PersistentHashMap_createNode(uint32_t shift, Object * key1, Object * val1, uint32_t hash2, Object * key2, Object * val2, BOOL *isNodeAdded); -HashMapNode *PersistentHashMapNode_assoc(HashMapNode *self, uint32_t shift, uint32_t hash, void *key, void *value); -HashMapNode *PersistentHashMapNode_dissoc(HashMapNode *self, uint32_t shift, uint32_t hash, void *key); -Object *PersistentHashMapNode_get(HashMapNode *self, uint32_t shift, uint32_t hash, void *key); +Object* PersistentHashMapNode_assoc(Object *self, uint32_t shift, uint32_t hash, Object * key, Object * value, BOOL *isNodeAdded); +PersistentHashMap *PersistentHashMapNode_dissoc(Object *self, uint32_t shift, uint32_t hash, void *key); +Object *PersistentHashMapNode_get(Object *self, uint32_t shift, uint32_t hash, void *key); -PersistentArrayMap* PersistentArrayMap_empty(); +PersistentHashMap* PersistentHashMap_empty(); -BOOL PersistentArrayMap_equals(PersistentArrayMap *self, PersistentArrayMap *other); -uint64_t PersistentArrayMap_hash(PersistentArrayMap *self); -String *PersistentArrayMap_toString(PersistentArrayMap *self); -void PersistentArrayMap_destroy(PersistentArrayMap *self, BOOL deallocateChildren); +BOOL PersistentHashMap_equals(PersistentHashMap * self, PersistentHashMap * other); +uint64_t PersistentHashMap_hash(PersistentHashMap *self); +String *PersistentHashMap_toString(PersistentHashMap *self); +void PersistentHashMap_destroy(PersistentHashMap *self, BOOL deallocateChildren); -PersistentArrayMap* PersistentArrayMap_assoc(PersistentArrayMap *self, void *key, void *value); -PersistentArrayMap *PersistentArrayMap_copy(PersistentArrayMap *self); -PersistentArrayMap* PersistentArrayMap_dissoc(PersistentArrayMap *self, void *key); -void* PersistentArrayMap_get(PersistentArrayMap *self, void *key); -void* PersistentArrayMap_dynamic_get(void *self, void *key); -int64_t PersistentArrayMap_indexOf(PersistentArrayMap *self, void *key); +PersistentHashMap* PersistentHashMap_assoc(PersistentHashMap *self, Object *key, Object *value); +PersistentHashMap *PersistentHashMap_copy(PersistentHashMap *self); +PersistentHashMap* PersistentHashMap_dissoc(PersistentHashMap *self, void *key); +void* PersistentHashMap_get(PersistentHashMap *self, void *key); +void* PersistentHashMap_dynamic_get(void *self, void *key); +int64_t PersistentHashMap_indexOf(PersistentHashMap *self, void *key); -PersistentArrayMap* PersistentArrayMap_create(); -PersistentArrayMap* PersistentArrayMap_createMany(uint64_t pairCount, ...); +PersistentHashMap* PersistentHashMap_create(); +PersistentHashMap* PersistentHashMap_createMany(uint64_t pairCount, ...); #endif diff --git a/backend/runtime/String.h b/backend/runtime/String.h index 4234813..10bb821 100644 --- a/backend/runtime/String.h +++ b/backend/runtime/String.h @@ -6,6 +6,7 @@ #include #include #include +#include #include "defines.h" typedef struct Object Object; diff --git a/backend/runtime/Transient.h b/backend/runtime/Transient.h index 312926e..f7e8e82 100644 --- a/backend/runtime/Transient.h +++ b/backend/runtime/Transient.h @@ -2,6 +2,7 @@ #define RT_TRANSIENT #include +#include #define PERSISTENT 0 diff --git a/backend/runtime/defines.h b/backend/runtime/defines.h index bfa75d4..0568eb7 100644 --- a/backend/runtime/defines.h +++ b/backend/runtime/defines.h @@ -11,21 +11,25 @@ typedef uint8_t BOOL; #define HASHTABLE_THRESHOLD 8 enum objectType { - integerType = 1, - stringType, - persistentListType, - persistentVectorType, - persistentVectorNodeType, - doubleType, - nilType, - booleanType, - symbolType, - concurrentHashMapType, - keywordType, - functionType, - bigIntegerType, - ratioType, - persistentArrayMapType, + integerType = 1, + stringType, + persistentListType, + persistentVectorType, + persistentVectorNodeType, + doubleType, + nilType, + booleanType, + symbolType, + concurrentHashMapType, + keywordType, + functionType, + bigIntegerType, + ratioType, + bitmapIndexedNodeType, + hashCollisionNodeType, + containerNodeType, + persistentHashMapType, + persistentArrayMapType, }; typedef enum objectType objectType; diff --git a/backend/runtime/runtime-tests.cpp b/backend/runtime/runtime-tests.cpp index e3f3d6c..542c798 100644 --- a/backend/runtime/runtime-tests.cpp +++ b/backend/runtime/runtime-tests.cpp @@ -13,389 +13,446 @@ #include #include #include +#include extern "C" { - typedef struct PersistentVectorNode PersistentVectorNode; - #include "defines.h" - - typedef struct Object { +typedef struct PersistentVectorNode PersistentVectorNode; +#include "defines.h" + +typedef struct Object { objectType type; - atomic_uint_fast64_t refCount; - } Object; - - typedef struct PersistentList { + std::atomic refCount; +} Object; + +typedef struct PersistentList { void *first; PersistentList *rest; uint64_t count; - } PersistentList; - - typedef struct PersistentVector { +} PersistentList; + +typedef struct PersistentVector { uint64_t count; uint64_t shift; PersistentVectorNode *tail; PersistentVectorNode *root; - } PersistentVector; +} PersistentVector; +typedef struct PersistentHashMap { + uint64_t count; + Object *root; + Object *nilValue; +} PersistentHashMap; - typedef struct Integer { +typedef struct Integer { int64_t value; - } Integer; +} Integer; - typedef struct Double { +typedef struct Double { double value; - } Double; - - char release(void *); - void retain(void *); - - PersistentList* PersistentList_create(void *first, PersistentList *rest); - PersistentList* PersistentList_conj(PersistentList *self, void *other); - - enum specialisedString { +} Double; + +char release(void *); +void retain(void *); + +PersistentList *PersistentList_create(void *first, PersistentList *rest); +PersistentList *PersistentList_conj(PersistentList *self, void *other); + +enum specialisedString { staticString, dynamicString, compoundString - }; +}; + +typedef enum specialisedString specialisedString; - typedef enum specialisedString specialisedString; - - struct String { +struct String { uint64_t count; uint64_t hash; specialisedString specialisation; - char value[]; - }; - - typedef struct String String; - - String *Object_toString(Object * self); - String *toString(void * self); - uint64_t hash(void * self); - - String *String_compactify(String *self); - char *String_c_str(String *self); - - - Integer *Integer_create(int64_t); - Double* Double_create(double d); - - Object *super(void *); - void *Object_data(Object *); - - PersistentVector* PersistentVector_conj(PersistentVector *self, void *other); - PersistentVector* PersistentVector_assoc(PersistentVector *self, uint64_t index, void *other); - void* PersistentVector_nth(PersistentVector *self, uint64_t index); - PersistentVector *PersistentVector_create(); - void initialise_memory(); - - typedef struct ConcurrentHashMapEntry { - void * _Atomic key; - void * _Atomic value; - _Atomic uint64_t keyHash; - _Atomic unsigned short leaps; - } ConcurrentHashMapEntry; - - typedef struct ConcurrentHashMapNode { + char value[]; +}; + +typedef struct String String; + +String *Object_toString(Object *self); +String *toString(void *self); +uint64_t hash(void *self); + +String *String_compactify(String *self); +char *String_c_str(String *self); + + +Integer *Integer_create(int64_t); +Double *Double_create(double d); + +Object *super(void *); +void *Object_data(Object *); + +PersistentVector *PersistentVector_conj(PersistentVector *self, void *other); +PersistentVector *PersistentVector_assoc(PersistentVector *self, uint64_t index, void *other); +void *PersistentVector_nth(PersistentVector *self, uint64_t index); +PersistentVector *PersistentVector_create(); +void initialise_memory(); + +typedef struct ConcurrentHashMapEntry { + std::atomic key; + std::atomic value; + std::atomic keyHash; + std::atomic leaps; +} ConcurrentHashMapEntry; + +typedef struct ConcurrentHashMapNode { uint64_t sizeMask; short int resizingThreshold; ConcurrentHashMapEntry array[]; - } ConcurrentHashMapNode; - - typedef struct ConcurrentHashMap { - ConcurrentHashMapNode * _Atomic root; - } ConcurrentHashMap; - - ConcurrentHashMap *ConcurrentHashMap_create(unsigned char initialSizeExponent); - - void ConcurrentHashMap_assoc(ConcurrentHashMap *self, void *key, void *value); - void ConcurrentHashMap_dissoc(ConcurrentHashMap *self, void *key); - void *ConcurrentHashMap_get(ConcurrentHashMap *self, void *key); - void PersistentVector_print(PersistentVector *self); +} ConcurrentHashMapNode; + +typedef struct ConcurrentHashMap { + std::atomic root; +} ConcurrentHashMap; + +ConcurrentHashMap *ConcurrentHashMap_create(unsigned char initialSizeExponent); + +void ConcurrentHashMap_assoc(ConcurrentHashMap *self, void *key, void *value); +void ConcurrentHashMap_dissoc(ConcurrentHashMap *self, void *key); +void *ConcurrentHashMap_get(ConcurrentHashMap *self, void *key); +void PersistentVector_print(PersistentVector *self); + +PersistentHashMap *PersistentHashMap_create(); +PersistentHashMap *PersistentHashMap_assoc(PersistentHashMap *self, Object *key, Object *value); } //#include -extern _Atomic uint64_t allocationCount[13]; +extern std::atomic allocationCount[18]; void pd() { - printf("Ref counters: %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu\n", allocationCount[0], allocationCount[1], allocationCount[2], allocationCount[3], allocationCount[4], allocationCount[5], allocationCount[6], allocationCount[7], allocationCount[8], allocationCount[9], allocationCount[10], allocationCount[11], allocationCount[12]); + printf("Ref counters: %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu\n", + allocationCount[0].load(), allocationCount[1].load(), allocationCount[2].load(), allocationCount[3].load(), + allocationCount[4].load(), allocationCount[5].load(), allocationCount[6].load(), allocationCount[7].load(), + allocationCount[8].load(), allocationCount[9].load(), allocationCount[10].load(), allocationCount[11].load(), + allocationCount[12].load(), allocationCount[13].load(), allocationCount[14].load(), allocationCount[15].load(), allocationCount[16].load(), allocationCount[17].load()); } typedef struct HashThreadParams { - int start; - int stop; - ConcurrentHashMap *map; + int start; + int stop; + ConcurrentHashMap *map; } HashThreadParams; void *startThread(void *param) { - HashThreadParams *p = (HashThreadParams *)param; - ConcurrentHashMap *l = p->map; - - for(int i=p->start; istop; i++) { - Integer *n = Integer_create(i); - retain(n); - ConcurrentHashMap_assoc(l, n, n); - } - - return NULL; + HashThreadParams *p = (HashThreadParams *) param; + ConcurrentHashMap *l = p->map; + + for (int i = p->start; i < p->stop; i++) { + Integer *n = Integer_create(i); + retain(n); + ConcurrentHashMap_assoc(l, n, n); + } + + return NULL; } -void testMap (bool pauses) { - ConcurrentHashMap *l = ConcurrentHashMap_create(28); - // // l = l->conj(new Number(3)); - // // l = l->conj(new Number(7)); - // // l = l->conj(new PersistentList(new Number(2))); - // // l = l->conj(new PersistentList()); - printf("Press a key to start\n"); - pd(); - if(pauses) getchar(); - struct timeval as, ap; - gettimeofday(&as, NULL); - - HashThreadParams params[10]; - pthread_t threads[10]; - - for(int i=0; i<10; i++) { - params[i].start = i * 10000000; - params[i].stop = (i+1) * 10000000; - params[i].map = l; - pthread_create(&(threads[i]), NULL, startThread, (void *) ¶ms[i]); - } - - for(int i=0; i<10; i++) pthread_join(threads[i], NULL); - - gettimeofday(&ap, NULL); - - printf("Time: %f\n", (ap.tv_sec - as.tv_sec) + (ap.tv_usec - as.tv_usec)/1000000.0); - - pd(); - - struct timeval ss, sp; - gettimeofday(&ss, NULL); - int64_t sum = 0; - Integer *k = Integer_create(1); - for(int i=0; i< 100000000; i++) { - k->value = i; - retain(k); - void *o = ConcurrentHashMap_get(l, k); - assert(o); - if(super(o)->type != integerType) { - retain(k); - printf("Unknown type %d for entry %s\n", super(o)->type, String_c_str(toString(k))); - retain(k); - o = ConcurrentHashMap_get(l, k); - retain(k); - printf("Unknown type %d for entry %s\n", super(o)->type, String_c_str(toString(k))); - retain(l); - retain(o); - printf("Contents: %s %s\n", String_c_str(toString(o)), String_c_str(String_compactify(toString(l)))); +void testMap(bool pauses) { + ConcurrentHashMap *l = ConcurrentHashMap_create(28); + // // l = l->conj(new Number(3)); + // // l = l->conj(new Number(7)); + // // l = l->conj(new PersistentList(new Number(2))); + // // l = l->conj(new PersistentList()); + printf("Press a key to start\n"); + pd(); + if (pauses) getchar(); + struct timeval as, ap; + gettimeofday(&as, NULL); + + HashThreadParams params[10]; + pthread_t threads[10]; + + for (int i = 0; i < 10; i++) { + params[i].start = i * 10000000; + params[i].stop = (i + 1) * 10000000; + params[i].map = l; + pthread_create(&(threads[i]), NULL, startThread, (void *) ¶ms[i]); } - assert(super(o)->type == integerType); - Integer *res = (Integer *) o; - assert(res->value == i); - sum += res->value; - release(res); - } - - - release(k); - gettimeofday(&sp, NULL); + for (int i = 0; i < 10; i++) pthread_join(threads[i], NULL); + + gettimeofday(&ap, NULL); + + printf("Time: %f\n", (ap.tv_sec - as.tv_sec) + (ap.tv_usec - as.tv_usec) / 1000000.0); + + pd(); + + struct timeval ss, sp; + gettimeofday(&ss, NULL); + int64_t sum = 0; + Integer *k = Integer_create(1); + for (int i = 0; i < 100000000; i++) { + k->value = i; + retain(k); + void *o = ConcurrentHashMap_get(l, k); + assert(o); + if (super(o)->type != integerType) { + retain(k); + printf("Unknown type %d for entry %s\n", super(o)->type, String_c_str(toString(k))); + retain(k); + o = ConcurrentHashMap_get(l, k); + retain(k); + printf("Unknown type %d for entry %s\n", super(o)->type, String_c_str(toString(k))); + retain(l); + retain(o); + printf("Contents: %s %s\n", String_c_str(toString(o)), String_c_str(String_compactify(toString(l)))); + } + + assert(super(o)->type == integerType); + Integer *res = (Integer *) o; + assert(res->value == i); + sum += res->value; + release(res); + } + + + release(k); + gettimeofday(&sp, NULL); // retain(l); // printf("Contents: %s \n", String_c_str(String_compactify(toString(l)))); - printf("Sum: %llu, Time: %f\n", sum, (sp.tv_sec - ss.tv_sec) + (sp.tv_usec - ss.tv_usec)/1000000.0); - assert(sum == 4999999950000000ULL && "Wrong result"); - - clock_t ds = clock(); - release(l); - clock_t dd = clock(); - printf("Release Time: %f\n", (double)(dd - ds) / CLOCKS_PER_SEC); - pd(); - // for (int i=0;i<100000000; i++) { - // Integer *n = Integer_create(i); - // PersistentList *k = PersistentList_conj(l, super(n)); - // l = k; - // } - // pd(); - // clock_t ap = clock(); - // printf("Array size: %llu\nRef count: %llu\nTime: %f\n", l->count, super(l)->refCount, (double)(ap - as) / CLOCKS_PER_SEC); - - // if (pauses) getchar(); - // clock_t os = clock(); - - // PersistentList *tmp = l; - // int64_t sum = 0; - // while(tmp != NULL) { - // if(tmp->first) sum += ((Integer *)(Object_data(tmp->first)))->value; - // tmp = tmp->rest; - // } - // clock_t op = clock(); - // printf("Sum: %llu\nTime: %f\n", sum, (double)(op - os) / CLOCKS_PER_SEC); - // if(pauses) getchar(); - // clock_t ds = clock(); - // printf("%llu\n", super(l)->refCount); - // release(l); - // pd(); - // clock_t dp = clock(); - // printf("Released\nTime: %f\n", (double)(dp - ds) / CLOCKS_PER_SEC); + printf("Sum: %llu, Time: %f\n", sum, (sp.tv_sec - ss.tv_sec) + (sp.tv_usec - ss.tv_usec) / 1000000.0); + assert(sum == 4999999950000000ULL && "Wrong result"); + + clock_t ds = clock(); + release(l); + clock_t dd = clock(); + printf("Release Time: %f\n", (double) (dd - ds) / CLOCKS_PER_SEC); + pd(); + // for (int i=0;i<100000000; i++) { + // Integer *n = Integer_create(i); + // PersistentList *k = PersistentList_conj(l, super(n)); + // l = k; + // } + // pd(); + // clock_t ap = clock(); + // printf("Array size: %llu\nRef count: %llu\nTime: %f\n", l->count, super(l)->refCount, (double)(ap - as) / CLOCKS_PER_SEC); + + // if (pauses) getchar(); + // clock_t os = clock(); + + // PersistentList *tmp = l; + // int64_t sum = 0; + // while(tmp != NULL) { + // if(tmp->first) sum += ((Integer *)(Object_data(tmp->first)))->value; + // tmp = tmp->rest; + // } + // clock_t op = clock(); + // printf("Sum: %llu\nTime: %f\n", sum, (double)(op - os) / CLOCKS_PER_SEC); + // if(pauses) getchar(); + // clock_t ds = clock(); + // printf("%llu\n", super(l)->refCount); + // release(l); + // pd(); + // clock_t dp = clock(); + // printf("Released\nTime: %f\n", (double)(dp - ds) / CLOCKS_PER_SEC); } - -void testList (bool pauses) { - PersistentList *l = PersistentList_create(NULL, NULL); - // l = l->conj(new Number(3)); - // l = l->conj(new Number(7)); - // l = l->conj(new PersistentList(new Number(2))); - // l = l->conj(new PersistentList()); - printf("Press a key to start\n"); - pd(); - if(pauses) getchar(); - clock_t as = clock(); - - for (int i=0;i<100000000; i++) { - Integer *n = Integer_create(i); - PersistentList *k = PersistentList_conj(l, n); - l = k; - } - pd(); - clock_t ap = clock(); - printf("Array size: %llu\nRef count: %llu\nTime: %f\n", l->count, super(l)->refCount, (double)(ap - as) / CLOCKS_PER_SEC); - - if (pauses) getchar(); - clock_t os = clock(); - - PersistentList *tmp = l; - int64_t sum = 0; - while(tmp != NULL) { - if(tmp->first) sum += ((Integer *)(tmp->first))->value; - tmp = tmp->rest; - } - assert(sum == 4999999950000000ull && "Wrong result"); - clock_t op = clock(); - printf("Sum: %llu\nTime: %f\n", sum, (double)(op - os) / CLOCKS_PER_SEC); - if(pauses) getchar(); - clock_t ds = clock(); - printf("%llu\n", super(l)->refCount); - release(l); - pd(); - clock_t dp = clock(); - printf("Released\nTime: %f\n", (double)(dp - ds) / CLOCKS_PER_SEC); +void testList(bool pauses) { + PersistentList *l = PersistentList_create(NULL, NULL); + // l = l->conj(new Number(3)); + // l = l->conj(new Number(7)); + // l = l->conj(new PersistentList(new Number(2))); + // l = l->conj(new PersistentList()); + printf("Press a key to start\n"); + pd(); + if (pauses) getchar(); + clock_t as = clock(); + + for (int i = 0; i < 100000000; i++) { + Integer *n = Integer_create(i); + PersistentList *k = PersistentList_conj(l, n); + l = k; + } + pd(); + clock_t ap = clock(); + printf("Array size: %llu\nRef count: %llu\nTime: %f\n", l->count, super(l)->refCount.load(), + (double) (ap - as) / CLOCKS_PER_SEC); + + if (pauses) getchar(); + clock_t os = clock(); + + PersistentList *tmp = l; + int64_t sum = 0; + while (tmp != NULL) { + if (tmp->first) sum += ((Integer *) (tmp->first))->value; + tmp = tmp->rest; + } + assert(sum == 4999999950000000ull && "Wrong result"); + clock_t op = clock(); + printf("Sum: %llu\nTime: %f\n", sum, (double) (op - os) / CLOCKS_PER_SEC); + if (pauses) getchar(); + clock_t ds = clock(); + printf("%llu\n", super(l)->refCount.load()); + release(l); + pd(); + clock_t dp = clock(); + printf("Released\nTime: %f\n", (double) (dp - ds) / CLOCKS_PER_SEC); } +void testPersistentHashMapAssoc(bool pauses) { + PersistentHashMap *l = PersistentHashMap_create(); + + printf("Press a key to start\n"); + pd(); + if (pauses) getchar(); + clock_t as = clock(); -void testVector (bool pauses, bool reuseSwitch = true) { - // printf("Total size: %lu %lu\n", sizeof(Object), sizeof(Integer)); - // printf("Total size: %lu %lu\n", sizeof(PersistentVector), sizeof(PersistentVectorNode)); - PersistentVector *l = PersistentVector_create(); - // l = l->conj(new Number(3)); - // l = l->conj(new Number(7)); - // l = l->conj(new PersistentList(new Number(2))); - // l = l->conj(new PersistentList()); - printf("Press a key to start\n"); - pd(); - if(pauses) getchar(); - clock_t as = clock(); - - bool reuse = reuseSwitch ? (rand() & 1) : false; - - for (int i=0;i<100000000; i++) { - // PersistentVector_print(l); - // printf("=======*****************==========="); - // fflush(stdout); - reuse = reuseSwitch ? (rand() & 1) : false; - Integer *n = Integer_create(i); - if(!reuse) retain(l); - PersistentVector *k = PersistentVector_conj(l, n); - if(!reuse) release(l); - - l = k; - // printf("%d\r", i); - // fflush(stdout); + for (int i = 0; i < 500000; i++) { + Integer *n = Integer_create(i); + retain(n); + + if (i == 31) { + printf("Ldsfmfsjsfdmjk\n"); + } + + PersistentHashMap *k = PersistentHashMap_assoc(l, super(n), super(n)); + retain(k); + String *s = toString(k); + + String *sComp = String_compactify(s); + char *text = String_c_str(sComp); + printf("Result: %s\n", text); + release(sComp); + l = k; + + + + + + } + + pd(); + clock_t ap = clock(); + printf("Map size: %llu\nRef count: %llu\nTime: %f\n", l->count, super(l)->refCount.load(), + (double) (ap - as) / CLOCKS_PER_SEC); + + release(l); + + pd(); +// if (pauses) getchar(); + +} + +void testVector(bool pauses, bool reuseSwitch = true) { + // printf("Total size: %lu %lu\n", sizeof(Object), sizeof(Integer)); + // printf("Total size: %lu %lu\n", sizeof(PersistentVector), sizeof(PersistentVectorNode)); + PersistentVector *l = PersistentVector_create(); + // l = l->conj(new Number(3)); + // l = l->conj(new Number(7)); + // l = l->conj(new PersistentList(new Number(2))); + // l = l->conj(new PersistentList()); + printf("Press a key to start\n"); + pd(); + if (pauses) getchar(); + clock_t as = clock(); + + bool reuse = reuseSwitch ? (rand() & 1) : false; + + for (int i = 0; i < 100000000; i++) { + // PersistentVector_print(l); + // printf("=======*****************==========="); + // fflush(stdout); + reuse = reuseSwitch ? (rand() & 1) : false; + Integer *n = Integer_create(i); + if (!reuse) retain(l); + PersistentVector *k = PersistentVector_conj(l, n); + if (!reuse) release(l); + + l = k; + // printf("%d\r", i); + // fflush(stdout); // PersistentVector_print(l); - } - pd(); - clock_t ap = clock(); - printf("Array size: %llu\nRef count: %llu\nTime: %f\n", l->count, super(l)->refCount, (double)(ap - as) / CLOCKS_PER_SEC); - - if (pauses) getchar(); - clock_t os = clock(); - - int64_t sum = 0; - - for(int i=0; i< l->count; i++) { - retain(l); - Integer *ob = (Integer *) PersistentVector_nth(l, i); - sum += (ob)->value; - release(ob); - } - - clock_t op = clock(); - printf("Sum: %llu\nTime: %f\n", sum, (double)(op - os) / CLOCKS_PER_SEC); - assert(sum == 4999999950000000ull && "Wrong result"); - if(pauses) getchar(); - int64_t sum2 = 0; - int64_t *array = (int64_t *)malloc(100000000*sizeof(int64_t)); - memset(array, 0, 100000000); - for(int i=0; i< 100000000; i++) { - array[i] = i; - } - - clock_t oss = clock(); - for(int i=0; i< l->count; i++) { - sum2 += array[i]; - } - clock_t opp = clock(); - free(array); - printf("Sum2: %llu\nTime: %f\n", sum2, (double)(opp - oss) / CLOCKS_PER_SEC); - - clock_t ass = clock(); - for (int i=0;i<100000000; i++) { - // PersistentVector_print(l); - // printf("=======*****************==========="); - // fflush(stdout); - reuse = reuseSwitch ? (rand() & 1) : false; - Integer *n = Integer_create(7); - if(!reuse) retain(l); - PersistentVector *k = PersistentVector_assoc(l, i, n); - if(!reuse) release(l); - l = k; + } + pd(); + clock_t ap = clock(); + printf("Array size: %llu\nRef count: %llu\nTime: %f\n", l->count, super(l)->refCount.load(), + (double) (ap - as) / CLOCKS_PER_SEC); + + if (pauses) getchar(); + clock_t os = clock(); + + int64_t sum = 0; + + for (int i = 0; i < l->count; i++) { + retain(l); + Integer *ob = (Integer *) PersistentVector_nth(l, i); + sum += (ob)->value; + release(ob); + } + + clock_t op = clock(); + printf("Sum: %llu\nTime: %f\n", sum, (double) (op - os) / CLOCKS_PER_SEC); + assert(sum == 4999999950000000ull && "Wrong result"); + if (pauses) getchar(); + int64_t sum2 = 0; + int64_t *array = (int64_t *) malloc(100000000 * sizeof(int64_t)); + memset(array, 0, 100000000); + for (int i = 0; i < 100000000; i++) { + array[i] = i; + } + + clock_t oss = clock(); + for (int i = 0; i < l->count; i++) { + sum2 += array[i]; + } + clock_t opp = clock(); + free(array); + printf("Sum2: %llu\nTime: %f\n", sum2, (double) (opp - oss) / CLOCKS_PER_SEC); + + clock_t ass = clock(); + for (int i = 0; i < 100000000; i++) { + // PersistentVector_print(l); + // printf("=======*****************==========="); + // fflush(stdout); + reuse = reuseSwitch ? (rand() & 1) : false; + Integer *n = Integer_create(7); + if (!reuse) retain(l); + PersistentVector *k = PersistentVector_assoc(l, i, n); + if (!reuse) release(l); + l = k; // printf("%d\r",i); - } - - sum = 0; - - for(int i=0; i< l->count; i++) { - retain(l); - Integer *ob = (Integer *) PersistentVector_nth(l, i); - sum += ob->value; - release(ob); - } - - clock_t asd = clock(); - printf("Assocs + sum: %llu\nTime: %f\n", sum, (double)(asd - ass) / CLOCKS_PER_SEC); - clock_t ds = clock(); - printf("%llu\n", super(l)->refCount); - release(l); - pd(); - clock_t dp = clock(); - printf("Released\nTime: %f\n", (double)(dp - ds) / CLOCKS_PER_SEC); -} + } + + sum = 0; + for (int i = 0; i < l->count; i++) { + retain(l); + Integer *ob = (Integer *) PersistentVector_nth(l, i); + sum += ob->value; + release(ob); + } + + clock_t asd = clock(); + printf("Assocs + sum: %llu\nTime: %f\n", sum, (double) (asd - ass) / CLOCKS_PER_SEC); + clock_t ds = clock(); + printf("%llu\n", super(l)->refCount.load()); + release(l); + pd(); + clock_t dp = clock(); + printf("Released\nTime: %f\n", (double) (dp - ds) / CLOCKS_PER_SEC); +} int main() { - srand(0); - initialise_memory(); - // for(int i=0; i<30; i++) testList(false); - // testList(false); - //// ProfilerStart("xx.prof"); + srand(0); + initialise_memory(); + // for(int i=0; i<30; i++) testList(false); + // testList(false); + //// ProfilerStart("xx.prof"); // testVector(false, true); - testMap(false); - // ProfilerStop(); - // getchar(); +// testMap(false); + testPersistentHashMapAssoc(false); + // ProfilerStop(); + // getchar(); } diff --git a/tests/assoc-persistent-hash-map.clj b/tests/assoc-persistent-hash-map.clj new file mode 100644 index 0000000..b9f8ec1 --- /dev/null +++ b/tests/assoc-persistent-hash-map.clj @@ -0,0 +1,4 @@ +(loop [m {} n 100] + (if (= 0 n) + m + (recur (assoc m [n] [n]) (- n 1)))) \ No newline at end of file From b634a56cfb69bf737f0602f40c1690de64d021d1 Mon Sep 17 00:00:00 2001 From: Mateusz Wejman Date: Wed, 5 Jun 2024 11:46:02 +0200 Subject: [PATCH 3/6] Created tests and dockerfile for runtime-tests compilation --- Dockerfile | 32 +++++++++ backend/runtime/BitmapIndexedNode.c | 74 ++++++++++++++------ backend/runtime/BitmapIndexedNode.h | 2 +- backend/runtime/ContainerNode.c | 29 ++++---- backend/runtime/Object.h | 10 +-- backend/runtime/PersistentHashMap.c | 103 +++++++++++++++++++++++++--- backend/runtime/PersistentHashMap.h | 2 + backend/runtime/runtime-tests.cpp | 89 +++++++++++++++++++----- 8 files changed, 273 insertions(+), 68 deletions(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b03d84e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +FROM ubuntu:mantic +LABEL authors="wejman" + +RUN apt-get update && apt-get install -y \ + procps llvm + +RUN apt install clang-16 lldb-16 lld-16 \ + libllvm-16-ocaml-dev libllvm16 llvm-16 llvm-16-dev llvm-16-doc \ + llvm-16-examples llvm-16-runtime build-essential -y + + + +RUN apt-get install valgrind -y + +RUN apt-get install leiningen -y + +RUN apt-get install cmake -y + +RUN apt-get install protobuf-compiler gmpc libgmp3-dev -y + +COPY . /app +WORKDIR /app + + +# compile the code in backend using CMakeLists.txt +RUN cmake ./backend/runtime -DCMAKE_PREFIX_PATH=/usr/bin/cmake -DCMAKE_BUILD_TYPE=Debug +RUN make -j8 + + + +# do nothing +ENTRYPOINT ["tail", "-f", "/dev/null"] \ No newline at end of file diff --git a/backend/runtime/BitmapIndexedNode.c b/backend/runtime/BitmapIndexedNode.c index d666c6f..25829bb 100644 --- a/backend/runtime/BitmapIndexedNode.c +++ b/backend/runtime/BitmapIndexedNode.c @@ -26,6 +26,7 @@ BitmapIndexedNode *BitmapIndexedNode_cloneAndSet(BitmapIndexedNode *nodeWithArra #endif uint32_t arraySize = __builtin_popcount(nodeWithArray->bitmap); Object *cloneSuper = allocate(sizeof(Object) + sizeof(BitmapIndexedNode) + sizeof(Object *) * arraySize * 2); + Object_create(cloneSuper, bitmapIndexedNodeType); BitmapIndexedNode *clone = Object_data(cloneSuper); memcpy(clone->array, nodeWithArray->array, sizeof(BitmapIndexedNode) * arraySize * 2); @@ -40,8 +41,6 @@ BitmapIndexedNode *BitmapIndexedNode_cloneAndSet(BitmapIndexedNode *nodeWithArra clone->array[idx] = a; clone->bitmap = nodeWithArray->bitmap; - Object_create(cloneSuper, bitmapIndexedNodeType); - release(nodeWithArray); #ifdef OBJECT_DEBUG @@ -61,9 +60,11 @@ BitmapIndexedNode_cloneAndSetTwo(BitmapIndexedNode *nodeWithArray, uint32_t idx, #ifdef OBJECT_DEBUG assert(a == NULL || a->magic == 0xdeadbeef && "Memory corruption!"); assert(a2 == NULL || a2->magic == 0xdeadbeef && "Memory corruption!"); + assert(super(nodeWithArray)->atomicRefCount > 0 && "Memory corruption!"); #endif uint32_t arraySize = __builtin_popcount(nodeWithArray->bitmap); Object *cloneSuper = allocate(sizeof(Object) + sizeof(BitmapIndexedNode) + sizeof(Object *) * arraySize * 2); + Object_create(cloneSuper, bitmapIndexedNodeType); BitmapIndexedNode *clone = Object_data(cloneSuper); memcpy(clone->array, nodeWithArray->array, sizeof(BitmapIndexedNode) * arraySize * 2); @@ -79,7 +80,7 @@ BitmapIndexedNode_cloneAndSetTwo(BitmapIndexedNode *nodeWithArray, uint32_t idx, clone->array[idx2] = a2; clone->bitmap = nodeWithArray->bitmap; - Object_create(cloneSuper, bitmapIndexedNodeType); + release(nodeWithArray); @@ -106,9 +107,9 @@ BitmapIndexedNode *BitmapIndexedNode_empty() { BitmapIndexedNode *BitmapIndexedNode_create() { size_t size = sizeof(Object) + sizeof(BitmapIndexedNode); Object *super = allocate(size); + Object_create(super, bitmapIndexedNodeType); BitmapIndexedNode *self = Object_data(super); self->bitmap = 0; - Object_create(super, bitmapIndexedNodeType); return self; } @@ -143,11 +144,16 @@ BitmapIndexedNode *BitmapIndexedNode_createVa(uint32_t bitmap, uint32_t arraySiz BitmapIndexedNode * BitmapIndexedNode_createWithInsertion(BitmapIndexedNode *oldSelf, uint32_t arraySize, uint32_t insertIndex, - Object *keyToInsert, Object *valueToInsert, uint32_t oldBitmap) { + Object *keyToInsert, Object *valueToInsert, uint32_t newBitmap) { +#ifdef OBJECT_DEBUG + assert(arraySize <= 16 && "Array size is too big!"); +#endif size_t newSize = sizeof(Object) + sizeof(BitmapIndexedNode) + sizeof(Object *) * arraySize * 2; Object *superValue = allocate(newSize); + memset(superValue, 0, newSize); + Object_create(superValue, bitmapIndexedNodeType); BitmapIndexedNode *self = Object_data(superValue); - self->bitmap = oldSelf->bitmap; + self->bitmap = newBitmap; if (arraySize > 1) { memcpy(self->array, oldSelf->array, sizeof(Object *) * insertIndex * 2); @@ -162,12 +168,11 @@ BitmapIndexedNode_createWithInsertion(BitmapIndexedNode *oldSelf, uint32_t array if (self->array[i] == NULL || i == 2 * insertIndex || i == 2 * insertIndex + 1) { continue; } + // Looks like there is a free of used keys somewhere Object_retain(self->array[i]); } - Object_create(superValue, bitmapIndexedNodeType); - oldSelf->bitmap = oldBitmap; release(oldSelf); #ifdef OBJECT_DEBUG @@ -200,7 +205,7 @@ Object *BitmapIndexedNode_assoc(BitmapIndexedNode *self, uint32_t shift, uint32_ #ifdef OBJECT_DEBUG assert(keyOrNull == NULL || keyOrNull->magic == 0xdeadbeef && "Memory corruption!"); - assert(valueOrNode || valueOrNode->magic == 0xdeadbeef && "Memory corruption!"); + assert(valueOrNode != NULL && valueOrNode->magic == 0xdeadbeef && "Memory corruption!"); #endif // Value is a node not a value @@ -216,6 +221,13 @@ Object *BitmapIndexedNode_assoc(BitmapIndexedNode *self, uint32_t shift, uint32_ if (keyOrNull == valueOrNode) { return super(self); } + + #ifdef OBJECT_DEBUG + void * ret = super(BitmapIndexedNode_cloneAndSet(self, 2 * idx + 1, keyOrNull)); + PersistentHashMapNode_check(ret, 1); + return ret; + #endif + return super(BitmapIndexedNode_cloneAndSet(self, 2 * idx + 1, keyOrNull)); } // Key is stored in this node @@ -226,9 +238,25 @@ Object *BitmapIndexedNode_assoc(BitmapIndexedNode *self, uint32_t shift, uint32_ return super(self); } Object_release(key); + + #ifdef OBJECT_DEBUG + void * ret = super(BitmapIndexedNode_cloneAndSet(self, 2 * idx + 1, value)); + PersistentHashMapNode_check(ret, 1); + return ret; + #endif + + return super(BitmapIndexedNode_cloneAndSet(self, 2 * idx + 1, value)); } *isNodeAdded = TRUE; + #ifdef OBJECT_DEBUG + void * ret = super(BitmapIndexedNode_cloneAndSetTwo( + self, 2 * idx, NULL, 2 * idx + 1, + PersistentHashMap_createNode( + shift + 5, keyOrNull, valueOrNode, hash, key, value, isNodeAdded))); + PersistentHashMapNode_check(ret, 1); + return ret; + #endif return super(BitmapIndexedNode_cloneAndSetTwo( self, 2 * idx, NULL, 2 * idx + 1, PersistentHashMap_createNode( @@ -237,17 +265,24 @@ Object *BitmapIndexedNode_assoc(BitmapIndexedNode *self, uint32_t shift, uint32_ uint8_t bitCount = __builtin_popcount(self->bitmap); if (bitCount >= 16) { uint32_t mask = BitmapIndexedNode_mask(hash, shift); + + #ifdef OBJECT_DEBUG + void * ret = ContainerNode_createFromBitmapIndexedNode(self, shift, hash, mask, key, value, isNodeAdded); + PersistentHashMapNode_check(ret, 1); + return ret; + #endif + return ContainerNode_createFromBitmapIndexedNode(self, shift, hash, mask, key, value, isNodeAdded); } else { - uint32_t oldBitmap = self->bitmap; - self->bitmap |= bit; + uint32_t newBitmap = self->bitmap | bit; BitmapIndexedNode *result = BitmapIndexedNode_createWithInsertion(self, bitCount + 1, idx, key, value, - oldBitmap); + newBitmap); + *isNodeAdded = TRUE; #ifdef OBJECT_DEBUG - assert(super(result)->magic == 0xdeadbeef && "Memory corruption!"); + assert(super(result)->magic == 0xdeadbeef && "Memory corruption!"); + PersistentHashMapNode_check(super(result), 1); #endif - *isNodeAdded = TRUE; return super(result); } } @@ -395,14 +430,9 @@ String *BitmapIndexedNode_toString(BitmapIndexedNode *self) { void BitmapIndexedNode_destroy(BitmapIndexedNode *self, BOOL deallocateChildren) { if (deallocateChildren) { - uint8_t idx = 0; - for (uint32_t i = 0; i < __builtin_popcount(self->bitmap); i++) { - if (self->bitmap & (1 << i)) { - if (self->array[2 * idx] != NULL) { - Object_release(self->array[2 * idx]); - } - Object_release(self->array[2 * idx + 1]); - idx += 1; + for (uint32_t i = 0; i < __builtin_popcount(self->bitmap) * 2; i++) { + if (self->array[i] != NULL) { + Object_release(self->array[i]); } } } diff --git a/backend/runtime/BitmapIndexedNode.h b/backend/runtime/BitmapIndexedNode.h index 0fe1d12..1ed57f3 100644 --- a/backend/runtime/BitmapIndexedNode.h +++ b/backend/runtime/BitmapIndexedNode.h @@ -38,7 +38,7 @@ HashMapNode* BitmapIndexedNode_dissoc(BitmapIndexedNode *self, uint32_t shift, u BitmapIndexedNode *BitmapIndexedNode_create(); BitmapIndexedNode *BitmapIndexedNode_createVa(uint32_t bitmap, uint32_t arraySize, ...); BitmapIndexedNode *BitmapIndexedNode_createWithout(BitmapIndexedNode *node, uint32_t bitmap, uint32_t idx); -BitmapIndexedNode *BitmapIndexedNode_createWithInsertion(BitmapIndexedNode *oldSelf, uint32_t arraySize, uint32_t insertIndex, Object *keyToInsert, Object *valueToInsert, uint32_t oldBitmap); +BitmapIndexedNode *BitmapIndexedNode_createWithInsertion(BitmapIndexedNode *oldSelf, uint32_t arraySize, uint32_t insertIndex, Object *keyToInsert, Object *valueToInsert, uint32_t newBitmap); BitmapIndexedNode *BitmapIndexedNode_empty(); diff --git a/backend/runtime/ContainerNode.c b/backend/runtime/ContainerNode.c index ec34cec..0202f2c 100644 --- a/backend/runtime/ContainerNode.c +++ b/backend/runtime/ContainerNode.c @@ -10,8 +10,15 @@ Object *ContainerNode_createFromBitmapIndexedNode(BitmapIndexedNode *node, uint3 uint32_t count = __builtin_popcount(node->bitmap); size_t size = sizeof(Object) + sizeof(ContainerNode) + 32 * sizeof(Object *); // Full size node Object *superValue = allocate(size); + Object_create(superValue, containerNodeType); ContainerNode *self = Object_data(superValue); self->count = count + 1; + + #ifdef OBJECT_DEBUG + assert(keyToInsert->magic == 0xdeadbeef && "Memory corruption!"); + assert(valueToInsert->magic == 0xdeadbeef && "Memory corruption!"); + #endif + self->array[insertIndex] = BitmapIndexedNode_assoc( BitmapIndexedNode_empty(), shift + 5, @@ -21,10 +28,6 @@ Object *ContainerNode_createFromBitmapIndexedNode(BitmapIndexedNode *node, uint3 isNodeAdded ); - #ifdef OBJECT_DEBUG - assert(keyToInsert->magic == 0xdeadbeef && "Memory corruption!"); - assert(valueToInsert->magic == 0xdeadbeef && "Memory corruption!"); - #endif uint32_t idx = 0; // NAPRAWIĆ TO, COŚ TU NIE DZIAŁA! for (uint32_t i = 0; i < 32; i++) { @@ -36,12 +39,16 @@ Object *ContainerNode_createFromBitmapIndexedNode(BitmapIndexedNode *node, uint3 self->array[i] = BitmapIndexedNode_assoc( BitmapIndexedNode_empty(), shift + 5, - hash(Object_data(node->array[2 * idx])), + Object_hash(node->array[2 * idx]), node->array[2 * idx], node->array[2 * idx + 1], isNodeAdded ); - } else if (node->array[2 * idx + 1] != NULL) { + } else { + #ifdef OBJECT_DEBUG + assert(node->array[2 * idx + 1] != NULL && "Value is NULL!"); + #endif + Object_retain(node->array[2 * idx + 1]); self->array[i] = node->array[2 * idx + 1]; } @@ -51,7 +58,6 @@ Object *ContainerNode_createFromBitmapIndexedNode(BitmapIndexedNode *node, uint3 } } - Object_create(superValue, containerNodeType); release(node); #ifdef OBJECT_DEBUG @@ -74,6 +80,7 @@ ContainerNode *ContainerNode_cloneAndSet(ContainerNode *nodeWithArray, uint32_t size_t newSize = sizeof(Object) + sizeof(ContainerNode) + 32 * sizeof(Object *); Object *superValue = allocate(newSize); + Object_create(superValue, containerNodeType); ContainerNode *self = Object_data(superValue); self->count = nodeWithArray->count; memcpy(self->array, nodeWithArray->array, 32 * @@ -85,7 +92,6 @@ ContainerNode *ContainerNode_cloneAndSet(ContainerNode *nodeWithArray, uint32_t Object_retain(self->array[i]); } self->array[idx] = a; - Object_create(superValue, containerNodeType); release(nodeWithArray); #ifdef OBJECT_DEBUG @@ -105,10 +111,6 @@ Object * ContainerNode_assoc(ContainerNode *self, uint32_t shift, uint32_t hash, Object *key, Object *value, BOOL *isNodeAdded) { uint32_t idx = BitmapIndexedNode_mask(hash, shift); - if (idx == 2) { - printf("Breakpoint here\n"); - } - Object *node = self->array[idx]; #ifdef OBJECT_DEBUG @@ -130,12 +132,13 @@ ContainerNode_assoc(ContainerNode *self, uint32_t shift, uint32_t hash, Object * ); } + Object_retain(node); Object *newNode = PersistentHashMapNode_assoc(node, shift + 5, hash, key, value, isNodeAdded); #ifdef OBJECT_DEBUG assert(newNode->magic == 0xdeadbeef && "Memory corruption!"); #endif if (newNode == node) { - Object_release(newNode); // Maybe? + Object_release(node); return super(self); } diff --git a/backend/runtime/Object.h b/backend/runtime/Object.h index de21481..39628f9 100644 --- a/backend/runtime/Object.h +++ b/backend/runtime/Object.h @@ -1,6 +1,5 @@ #ifndef RT_OBJECT #define RT_OBJECT -#define OBJECT_DEBUG 1 #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wnullability-completeness" @@ -78,9 +77,9 @@ inline void deallocate(void * restrict ptr) { } inline void *Object_data(Object * restrict self) { -//#ifdef OBJECT_DEBUG -// assert(self->magic == 0xdeadbeef && "Memory corruption!"); -//#endif +#ifdef OBJECT_DEBUG + assert(self->magic == 0xdeadbeef && "Memory corruption!"); +#endif return self + 1; } @@ -395,6 +394,9 @@ inline String *Object_toString(Object * restrict self) { } inline String *toString(void * restrict self) { +#ifdef OBJECT_DEBUG + assert(super(self)->magic == 0xdeadbeef && "Memory corruption!"); +#endif return Object_toString(super(self)); } diff --git a/backend/runtime/PersistentHashMap.c b/backend/runtime/PersistentHashMap.c index cb6c06a..753bf1a 100644 --- a/backend/runtime/PersistentHashMap.c +++ b/backend/runtime/PersistentHashMap.c @@ -18,9 +18,9 @@ PersistentHashMap *PersistentHashMap_empty() { PersistentHashMap *PersistentHashMap_create() { size_t size = sizeof(Object) + sizeof(PersistentHashMap); Object *superVal = allocate(size); - PersistentHashMap *self = Object_data(superVal); memset(superVal, 0, size); Object_create(superVal, persistentHashMapType); + PersistentHashMap *self = Object_data(superVal); return self; } @@ -106,11 +106,13 @@ String *PersistentHashMap_toString(PersistentHashMap *const self) { /* outside refcount system */ void PersistentHashMap_destroy(PersistentHashMap *self, BOOL deallocateChildren) { - if (self->root != NULL) { - Object_destroy(self->root, deallocateChildren); - } - if (self->nilValue != NULL) { - Object_destroy(self->nilValue, deallocateChildren); + if (deallocateChildren) { + if (self->root != NULL) { + Object_release(self->root); + } + if (self->nilValue != NULL) { + Object_release(self->nilValue); + } } } @@ -128,6 +130,7 @@ void PersistentHashMap_destroy(PersistentHashMap *self, BOOL deallocateChildren) PersistentHashMap *PersistentHashMap_copy(PersistentHashMap *self) { size_t size = sizeof(Object) + sizeof(PersistentHashMap); Object *superVal = allocate(size); + Object_create(superVal, persistentHashMapType); PersistentHashMap *new = Object_data(superVal); if (self->root != NULL) { Object_retain(self->root); @@ -143,7 +146,7 @@ PersistentHashMap *PersistentHashMap_copy(PersistentHashMap *self) { new->root = self->root; new->count = self->count; - Object_create(superVal, persistentHashMapType); + release(self); return new; @@ -193,8 +196,9 @@ PersistentHashMap *PersistentHashMap_assoc(PersistentHashMap *const self, Object return new; } -Object *PersistentHashMapNode_assoc(Object *const self, uint32_t shift, uint32_t hash, Object *const key, Object *const value, - BOOL *isNodeAdded) { +Object * +PersistentHashMapNode_assoc(Object *const self, uint32_t shift, uint32_t hash, Object *const key, Object *const value, + BOOL *isNodeAdded) { // NULL should not be a case here for self #ifdef OBJECT_DEBUG assert(self->magic == 0xdeadbeef && "Memory corruption!"); @@ -230,8 +234,9 @@ void *PersistentHashMap_dynamic_get(void *self, void *key) { return NULL; } -Object *PersistentHashMap_createNode(uint32_t shift, Object *const key1, Object *const val1, uint32_t hash2, Object *const key2, - Object *const val2, BOOL *isNodeAdded) { +Object * +PersistentHashMap_createNode(uint32_t shift, Object *const key1, Object *const val1, uint32_t hash2, Object *const key2, + Object *const val2, BOOL *isNodeAdded) { uint32_t hash1 = Object_hash(key1); if (hash1 == hash2) { return super(HashCollisionNode_create(hash1, 2, key1, val1, key2, val2)); @@ -245,3 +250,79 @@ Object *PersistentHashMap_createNode(uint32_t shift, Object *const key1, Object return BitmapIndexedNode_assoc(node, shift, hash2, key2, val2, isNodeAdded); } +void PersistentHashMapNode_check(Object *child, uint32_t expectedRefCount); + +void PersistentHashMap_childrenCheck(PersistentHashMap *map, uint32_t expectedRefCount) { + if (map->root != NULL) { + PersistentHashMapNode_check(map->root, expectedRefCount); + } +} + +void BitmapIndexedNode_check(Object *child, uint32_t expectedRefCount) { +#ifdef OBJECT_DEBUG + assert(child->magic == 0xdeadbeef && "Memory corruption!"); +#endif + BitmapIndexedNode *node = Object_data(child); + uint8_t idx = 0; + for (uint32_t i = 0; i < __builtin_popcount(node->bitmap); i++) { + if (node->bitmap & (1 << i)) { + if (node->array[2 * idx] != NULL) { + PersistentHashMapNode_check(node->array[2 * idx], expectedRefCount); + } + PersistentHashMapNode_check(node->array[2 * idx + 1], expectedRefCount); + idx += 1; + } + } +} + +void ContainerNode_check(Object *child, uint32_t expectedRefCount) { +#ifdef OBJECT_DEBUG + assert(child->magic == 0xdeadbeef && "Memory corruption!"); +#endif + ContainerNode *node = Object_data(child); + for (uint32_t i = 0; i < 32; i++) { + if (node->array[i] != NULL) { + PersistentHashMapNode_check(node->array[i], expectedRefCount); + } + } +} + +void HashCollisionNode_check(Object *child, uint32_t expectedRefCount) { +#ifdef OBJECT_DEBUG + assert(child->magic == 0xdeadbeef && "Memory corruption!"); +#endif + HashCollisionNode *node = Object_data(child); + for (uint32_t i = 0; i < node->count * 2; i++) { + if (node->array[i] == NULL) { + assert(FALSE && "Should not happen for HashCollisionNode_check"); + } + PersistentHashMapNode_check(node->array[i], expectedRefCount); + } +} + +void PersistentHashMapNode_check(Object *child, uint32_t expectedRefCount) { +#ifdef OBJECT_DEBUG + assert(child->magic == 0xdeadbeef && "Memory corruption!"); +#endif + switch (child->type) { + case bitmapIndexedNodeType: +// assert(child->atomicRefCount == expectedRefCount && "Wrong ref count for BitmapIndexedNode"); + BitmapIndexedNode_check(child, expectedRefCount); + break; + case containerNodeType: +// assert(child->atomicRefCount == expectedRefCount && "Wrong ref count for BitmapIndexedNode"); + ContainerNode_check(child, expectedRefCount); + break; + case hashCollisionNodeType: +// assert(child->atomicRefCount == expectedRefCount && "Wrong ref count for BitmapIndexedNode"); + HashCollisionNode_check(child, expectedRefCount); + break; + default: +// assert(child->atomicRefCount == 2 && "Wrong ref count for BitmapIndexedNode"); + if (child->type > 30) { + assert(FALSE && "Should not happen for PersistentHashMapNode_check"); + } + return; + } +} + diff --git a/backend/runtime/PersistentHashMap.h b/backend/runtime/PersistentHashMap.h index aa93aed..022af4a 100644 --- a/backend/runtime/PersistentHashMap.h +++ b/backend/runtime/PersistentHashMap.h @@ -35,6 +35,8 @@ BOOL PersistentHashMap_equals(PersistentHashMap * self, PersistentHashMap * othe uint64_t PersistentHashMap_hash(PersistentHashMap *self); String *PersistentHashMap_toString(PersistentHashMap *self); void PersistentHashMap_destroy(PersistentHashMap *self, BOOL deallocateChildren); +void PersistentHashMapNode_check(Object *child, uint32_t expectedRefCount); +void PersistentHashMap_childrenCheck(PersistentHashMap *map, uint32_t expectedRefCount); diff --git a/backend/runtime/runtime-tests.cpp b/backend/runtime/runtime-tests.cpp index 542c798..213e02e 100644 --- a/backend/runtime/runtime-tests.cpp +++ b/backend/runtime/runtime-tests.cpp @@ -13,15 +13,25 @@ #include #include #include +#ifdef ATOMIC_FIX #include +#endif extern "C" { typedef struct PersistentVectorNode PersistentVectorNode; #include "defines.h" + typedef struct Object { +#ifdef OBJECT_DEBUG + uint64_t magic; +#endif objectType type; +#ifdef ATOMIC_FIX std::atomic refCount; +#else + atomic_uint_fast64_t refCount; +#endif } Object; typedef struct PersistentList { @@ -96,10 +106,17 @@ PersistentVector *PersistentVector_create(); void initialise_memory(); typedef struct ConcurrentHashMapEntry { +#ifdef ATOMIC_FIX std::atomic key; std::atomic value; std::atomic keyHash; std::atomic leaps; +#else + void * _Atomic key; + void * _Atomic value; + _Atomic uint64_t keyHash; + _Atomic unsigned short leaps; +#endif } ConcurrentHashMapEntry; typedef struct ConcurrentHashMapNode { @@ -109,7 +126,11 @@ typedef struct ConcurrentHashMapNode { } ConcurrentHashMapNode; typedef struct ConcurrentHashMap { +#ifdef ATOMIC_FIX std::atomic root; +#else + ConcurrentHashMapNode * _Atomic root; +#endif } ConcurrentHashMap; ConcurrentHashMap *ConcurrentHashMap_create(unsigned char initialSizeExponent); @@ -121,18 +142,31 @@ void PersistentVector_print(PersistentVector *self); PersistentHashMap *PersistentHashMap_create(); PersistentHashMap *PersistentHashMap_assoc(PersistentHashMap *self, Object *key, Object *value); +void PersistentHashMap_childrenCheck(PersistentHashMap *self, uint32_t expectedRefCount); } //#include +#ifdef ATOMIC_FIX extern std::atomic allocationCount[18]; +#else +extern _Atomic uint64_t allocationCount[13]; +#endif void pd() { printf("Ref counters: %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu\n", +#ifdef ATOMIC_FIX allocationCount[0].load(), allocationCount[1].load(), allocationCount[2].load(), allocationCount[3].load(), allocationCount[4].load(), allocationCount[5].load(), allocationCount[6].load(), allocationCount[7].load(), allocationCount[8].load(), allocationCount[9].load(), allocationCount[10].load(), allocationCount[11].load(), - allocationCount[12].load(), allocationCount[13].load(), allocationCount[14].load(), allocationCount[15].load(), allocationCount[16].load(), allocationCount[17].load()); + allocationCount[12].load(), allocationCount[13].load(), allocationCount[14].load(), allocationCount[15].load(), allocationCount[16].load(), allocationCount[17].load() +#else + allocationCount[0], allocationCount[1], allocationCount[2], allocationCount[3], + allocationCount[4], allocationCount[5], allocationCount[6], allocationCount[7], + allocationCount[8], allocationCount[9], allocationCount[10], allocationCount[11], + allocationCount[12], allocationCount[13], allocationCount[14], allocationCount[15], allocationCount[16], allocationCount[17] +#endif + ); } @@ -292,7 +326,13 @@ void testList(bool pauses) { printf("Sum: %llu\nTime: %f\n", sum, (double) (op - os) / CLOCKS_PER_SEC); if (pauses) getchar(); clock_t ds = clock(); - printf("%llu\n", super(l)->refCount.load()); + printf("%llu\n", +#ifdef ATOMIC_FIX + super(l)->refCount.load() +#else + super(l)->refCount +#endif + ); release(l); pd(); clock_t dp = clock(); @@ -307,22 +347,20 @@ void testPersistentHashMapAssoc(bool pauses) { if (pauses) getchar(); clock_t as = clock(); - for (int i = 0; i < 500000; i++) { + for (int i = 0; i < 100000; i++) { Integer *n = Integer_create(i); retain(n); - - if (i == 31) { - printf("Ldsfmfsjsfdmjk\n"); - } - PersistentHashMap *k = PersistentHashMap_assoc(l, super(n), super(n)); - retain(k); - String *s = toString(k); +// PersistentHashMap_childrenCheck(k, 1); + - String *sComp = String_compactify(s); - char *text = String_c_str(sComp); - printf("Result: %s\n", text); - release(sComp); +// retain(k); +// String *s = toString(k); +// +// String *sComp = String_compactify(s); +// char *text = String_c_str(sComp); +// printf("Result: %s\n", text); +// release(sComp); l = k; @@ -333,7 +371,13 @@ void testPersistentHashMapAssoc(bool pauses) { pd(); clock_t ap = clock(); - printf("Map size: %llu\nRef count: %llu\nTime: %f\n", l->count, super(l)->refCount.load(), + printf("Map size: %llu\nRef count: %llu\nTime: %f\n", + l->count, +#ifdef ATOMIC_FIX + super(l)->refCount.load(), +#else + super(l)->refCount, +#endif (double) (ap - as) / CLOCKS_PER_SEC); release(l); @@ -375,7 +419,12 @@ void testVector(bool pauses, bool reuseSwitch = true) { } pd(); clock_t ap = clock(); - printf("Array size: %llu\nRef count: %llu\nTime: %f\n", l->count, super(l)->refCount.load(), + printf("Array size: %llu\nRef count: %llu\nTime: %f\n", l->count, +#ifdef ATOMIC_FIX + super(l)->refCount.load(), +#else + super(l)->refCount, +#endif (double) (ap - as) / CLOCKS_PER_SEC); if (pauses) getchar(); @@ -435,7 +484,13 @@ void testVector(bool pauses, bool reuseSwitch = true) { clock_t asd = clock(); printf("Assocs + sum: %llu\nTime: %f\n", sum, (double) (asd - ass) / CLOCKS_PER_SEC); clock_t ds = clock(); - printf("%llu\n", super(l)->refCount.load()); + printf("%llu\n", +#ifdef ATOMIC_FIX + super(l)->refCount.load() +#else + super(l)->refCount +#endif + ); release(l); pd(); clock_t dp = clock(); From 01a8209ead27f15abe744e26b88dadc1b446963f Mon Sep 17 00:00:00 2001 From: a-wiacek <33523701+a-wiacek@users.noreply.github.com> Date: Thu, 6 Jun 2024 10:06:19 +0200 Subject: [PATCH 4/6] Fix missing retains and overwriting values --- backend/runtime/BitmapIndexedNode.c | 4 +++- backend/runtime/ContainerNode.c | 4 ++-- backend/runtime/PersistentHashMap.c | 4 +++- backend/runtime/runtime-tests.cpp | 4 +++- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/backend/runtime/BitmapIndexedNode.c b/backend/runtime/BitmapIndexedNode.c index 25829bb..4f16aeb 100644 --- a/backend/runtime/BitmapIndexedNode.c +++ b/backend/runtime/BitmapIndexedNode.c @@ -257,6 +257,8 @@ Object *BitmapIndexedNode_assoc(BitmapIndexedNode *self, uint32_t shift, uint32_ PersistentHashMapNode_check(ret, 1); return ret; #endif + Object_retain(keyOrNull); + Object_retain(valueOrNode); return super(BitmapIndexedNode_cloneAndSetTwo( self, 2 * idx, NULL, 2 * idx + 1, PersistentHashMap_createNode( @@ -436,4 +438,4 @@ void BitmapIndexedNode_destroy(BitmapIndexedNode *self, BOOL deallocateChildren) } } } -} \ No newline at end of file +} diff --git a/backend/runtime/ContainerNode.c b/backend/runtime/ContainerNode.c index 0202f2c..34d8a1e 100644 --- a/backend/runtime/ContainerNode.c +++ b/backend/runtime/ContainerNode.c @@ -53,7 +53,7 @@ Object *ContainerNode_createFromBitmapIndexedNode(BitmapIndexedNode *node, uint3 self->array[i] = node->array[2 * idx + 1]; } idx += 1; - } else { + } else if (i != insertIndex) { self->array[i] = super(BitmapIndexedNode_empty()); } } @@ -235,4 +235,4 @@ void ContainerNode_destroy(ContainerNode *self, BOOL deallocateChildren) { } } } -} \ No newline at end of file +} diff --git a/backend/runtime/PersistentHashMap.c b/backend/runtime/PersistentHashMap.c index 753bf1a..9716601 100644 --- a/backend/runtime/PersistentHashMap.c +++ b/backend/runtime/PersistentHashMap.c @@ -172,9 +172,10 @@ PersistentHashMap *PersistentHashMap_assoc(PersistentHashMap *const self, Object if (root == NULL) { root = super(BitmapIndexedNode_empty()); + } else { + Object_retain(root); } - Object_retain(root); Object *newRoot = PersistentHashMapNode_assoc( root, 0, @@ -189,6 +190,7 @@ PersistentHashMap *PersistentHashMap_assoc(PersistentHashMap *const self, Object } PersistentHashMap *new = PersistentHashMap_copy(self); + if (new->root) Object_release(new->root); new->root = newRoot; if (isNodeAdded) { new->count += 1; diff --git a/backend/runtime/runtime-tests.cpp b/backend/runtime/runtime-tests.cpp index 213e02e..8cfa7b2 100644 --- a/backend/runtime/runtime-tests.cpp +++ b/backend/runtime/runtime-tests.cpp @@ -7,14 +7,16 @@ #include #include #include -#include #include #include #include #include #include + #ifdef ATOMIC_FIX #include +#else +#include #endif extern "C" { From 7f4b8abd5265d48459d2e73a16b35761fe6fe465 Mon Sep 17 00:00:00 2001 From: a-wiacek <33523701+a-wiacek@users.noreply.github.com> Date: Thu, 13 Jun 2024 15:10:21 +0200 Subject: [PATCH 5/6] Add map tests (in clj) --- tests/hashmap.clj | 60 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 tests/hashmap.clj diff --git a/tests/hashmap.clj b/tests/hashmap.clj new file mode 100644 index 0000000..1f5130d --- /dev/null +++ b/tests/hashmap.clj @@ -0,0 +1,60 @@ +(def big-map + (loop [m {} n 0] + (if (= n 1000000) + m + (recur (assoc m n [n]) (+ n 1))))) + +;; should be false +(loop [n 0] + (if (= n 1000000) + false + (let [v (get big-map n)] + (if (= v [n]) + (recur (+ n 1)) + n)))) + +;; should be nil +(get big-map 1234567) + +(def big-map2 + (loop [m {} n 999999] + (if (= n -1) + m + (recur (assoc m n [n]) (- n 1))))) + +;; should be true +(= big-map big-map2) + +(def big-map3 + (loop [m big-map n 0] + (if (= n 1000000) + m + (recur (dissoc m n) (+ n 1))))) + +;; should be true +(= big-map3 {}) + +(def big-map4 + (loop [m big-map n 1] + (if (= n 1000000) + m + (recur (assoc m n [n]) (+ n 1))))) + +;; should be false +(= big-map4 (dissoc big-map 0)) + +;; should be true +(= {} (dissoc (assoc {} 1 1) 1)) + +;; should be true (dissoc non-existent key) +(= {1 1} (dissoc {1 1} 2)) + +;; TODO: Find two non-equal values with equal hashes +(def h1 123) +(def h2 123) + +;; should be true +(= (assoc (assoc {} h1 h1) h2 h2) (assoc (assoc {} h2 h2) h1 h1)) + +;; should be true +(= (dissoc (assoc (assoc {} h1 h1) h2 h2) h1) (assoc {} h2 h2)) From 64e210437391fd265f93f168d6fe39ec75891f0f Mon Sep 17 00:00:00 2001 From: a-wiacek <33523701+a-wiacek@users.noreply.github.com> Date: Mon, 15 Jul 2024 10:52:54 +0200 Subject: [PATCH 6/6] Add contains tests --- tests/hashmap.clj | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/hashmap.clj b/tests/hashmap.clj index 1f5130d..f4bdbeb 100644 --- a/tests/hashmap.clj +++ b/tests/hashmap.clj @@ -58,3 +58,15 @@ ;; should be true (= (dissoc (assoc (assoc {} h1 h1) h2 h2) h1) (assoc {} h2 h2)) + +;; should be true +(contains? big-map 1) + +;; should be false +(contains? big-map 1234567) + +;; should be false +(contains? big-map3 1) + +;; should be false +(contains? big-map3 1234567)