put strings on a hash table, serializiation for test purposes

Qrakhen.Qamp.Core/Collections/Register.cs

@@ -1,4 +1,5 @@
 using System.Collections;
+using System.Diagnostics.CodeAnalysis;
 
 namespace Qrakhen.Qamp.Core.Collections;
 
@@ -26,7 +27,7 @@ public class Register<TKey, TValue> :
         return key;
     }
 
-    public bool TryGet(TKey key, out TValue? value) => _data.TryGetValue(key, out value);
+    public bool TryGet(TKey key, [MaybeNullWhen(false)] out TValue value) => _data.TryGetValue(key, out value);
     public TValue Get(TKey key) => _data[key];
     public void Set(TKey key, TValue value) => _data[key] = value;
     public bool Has(TKey key) => _data.ContainsKey(key);

Qrakhen.Qamp.Core/Execution/Runner.cs

@@ -60,7 +60,8 @@ public class Runner : IDisposable
         using Reader reader = new Reader(stream);
         Compilation.Digester digester = new(reader);
         Function function = digester.Digest();
-        File.WriteAllBytes("func.sqi", function.Segment.Serialize());
+        if (stream.Length > 64)
+            File.WriteAllBytes($"./func.{DateTime.Now:yyyyMMdd_HHmmss}.sqi", function.Serialize(true));
         Context closure = new Context(function);
         Push(Obj.Create(closure));
         Invoke(closure, 0);

Qrakhen.Qamp.Core/Execution/Segment.cs

@@ -1,5 +1,6 @@
 using Qrakhen.Qamp.Core.Collections;
 using Qrakhen.Qamp.Core.Values;
+using Qrakhen.Qamp.Core.Values.Objects;
 
 namespace Qrakhen.Qamp.Core.Execution;
 
@@ -59,20 +60,21 @@ public class Segment(
     public byte[] Serialize()
     {
         var opCodes = Instructions.Select(i => (byte)i).ToArray();
-        var constants = Constants.Select(c => c.Unsigned.GetBytes()).ToArray();
+        var constants = Constants.ToArray();
-        byte[] bytes = new byte[sizeof(long) * 2 + opCodes.Length + constants.Length * 8];
+        byte[] bytes = new byte[sizeof(long) + opCodes.Length];
-        Array.Copy(opCodes.LongLength.GetBytes(), 0, bytes, 0, sizeof(long));
+        System.Array.Copy(opCodes.LongLength.GetBytes(), 0, bytes, 0, sizeof(long));
-        Array.Copy((constants.LongLength * sizeof(long)).GetBytes(), 0, bytes, sizeof(long), sizeof(long));
+        System.Array.Copy(opCodes, 0, bytes, sizeof(long), opCodes.Length);
-        int offset = sizeof(long) * 2;
+        var list = new List<byte>(bytes);
-        for (long i = 0; i < opCodes.LongLength; i++) {
+        foreach (var constant in constants) {
-            bytes[offset++] = opCodes[i];
+            var data = constant.Signed.GetDynamicBytes();
-        }
+            if (data.Length == 1)
-        for (long i = 0; i < constants.LongLength; i++) {
+                list.Add(data[0]);
-            for (int j = 0; j < sizeof(long); j++) {
+            else {
-                bytes[offset++] = constants[i][j];
+                list.Add((byte)(data.Length | 0x80));
+                list.AddRange(data);
             }
         }
-        return bytes;
+        return list.ToArray();
     }
 
     public static Segment Deserialize(byte[] data)

Qrakhen.Qamp.Core/Extensions.cs

@@ -1,10 +1,25 @@
-using System.Text.Json.Serialization;
+using System.Text;
 using System.Text.RegularExpressions;
 
 namespace Qrakhen.Qamp.Core;
 
 public static class Extensions
 {
+    public static uint GetHash(this string str)
+    {
+        if (string.IsNullOrEmpty(str))
+            return 0;
+        byte[] bytes = Encoding.ASCII.GetBytes(str);
+        unchecked {
+            uint hash = 216613661u;
+            for (int i = 0; i < bytes.Length; i++) {
+                hash ^= bytes[i];
+                hash *= 16777619;
+            }
+            return hash;
+        }
+    }
+
     public static bool TryMatch(this Regex regex, string str, out Match match)
     {
         match = regex.Match(str);

Qrakhen.Qamp.Core/Tokenization/Reader.cs

@@ -17,43 +17,52 @@ internal static partial class ReaderPatterns
     public static partial Regex IsHexadecimal();
 
     public static readonly Dictionary<string, TokenType> Keywords = new();
+    public static readonly Dictionary<string, string> Aliases = new();
+
+    private static void Define(string key, TokenType type)
+    {
+        // not ideal, but momentarily better than setting up complex dialects
+        if (Keywords.ContainsValue(type))
+            Aliases[key] = Keywords.First(v => v.Value == type).Key;        
+        Keywords[key] = type;
+    }
 
     static ReaderPatterns()
     {
-        Keywords["false"]       = False;
+        Define("false",    False);
-        Keywords["true"]        = True;
+        Define("true",     True);
-        Keywords["null"]        = Null;
+        Define("null",     Null);
-        Keywords["void"]        = Null;
+        Define("void",     Null);
-        Keywords["and"]         = And;
+        Define("and",      And);
-        Keywords["else"]        = Else;
+        Define("else",     Else);
-        Keywords["for"]         = For;
+        Define("for",      For);
-        Keywords["if"]          = If;
+        Define("if",       If);
-        Keywords["or"]          = Or;
+        Define("or",       Or);
-        Keywords["this"]        = This;
+        Define("this",     This);
-        Keywords[".~"]          = This;
+        Define(".~",       This);
-        Keywords["var"]         = Var;
+        Define("var",      Var);
-        Keywords["*~"]          = Var;
+        Define("*~",       Var);
-        Keywords["while"]       = While;
+        Define("while",    While);
-        Keywords["do"]          = Do;
+        Define("do",       Do);
-        Keywords["ref"]         = Ref;
+        Define("ref",      Ref);
-        Keywords["function"]    = Function;
+        Define("function", Function);
-        Keywords["funqtion"]    = Function;
+        Define("funqtion", Function);
-        Keywords["fq"]          = Function;
+        Define("fq",       Function);
-        Keywords["funq"]        = Function;
+        Define("funq",     Function);
-        Keywords["return"]      = Return;
+        Define("return",   Return);
-        Keywords["<:"]          = Return;
+        Define("<:",       Return);
-        Keywords["class"]       = Class;
+        Define("class",    Class);
-        Keywords["base"]        = Base;
+        Define("base",     Base);
-        Keywords["^~"]          = Base;
+        Define("^~",       Base);
-        Keywords["typeof"]      = TypeOf;
+        Define("typeof",   TypeOf);
-        Keywords["?:"]          = TypeOf;
+        Define("?:",       TypeOf);
-        Keywords["print"]       = Print;
+        Define("print",    Print);
-        Keywords["::"]          = Var;
+        Define("::",       Print);
-        Keywords["globals"]     = PrintGlobals;
+        Define("globals",  PrintGlobals);
-        Keywords["stack"]       = PrintStack;
+        Define("stack",    PrintStack);
-        Keywords["expr"]        = PrintExpr;
+        Define("expr",     PrintExpr);
-        Keywords["import"]      = Import;
+        Define("import",   Import);
-        Keywords["export"]      = Export;
+        Define("export",   Export);
     }
 }
 

Qrakhen.Qamp.Core/Values/Objects/Function.cs

@@ -1,4 +1,5 @@
 using Qrakhen.Qamp.Core.Execution;
+using System.Text;
 
 namespace Qrakhen.Qamp.Core.Values.Objects;
 
@@ -10,4 +11,24 @@ public class Function(string name, Segment segment, int outerCount, int argument
     public readonly int ArgumentCount = argumentCount;
 
     public override string ToString() => $"{Name}()";
+
+    public byte[] Serialize(bool topLevel = false)
+    {
+        List<byte> result =
+        [
+            .. string.IsNullOrEmpty(Name) ? [] : Encoding.ASCII.GetBytes(Name),
+            0,
+            (byte)ArgumentCount,
+            .. OuterCount.GetBytes(),
+            .. Segment.Serialize()
+        ];
+        if (topLevel) {
+            result.AddRange([0xFF, 0xF0]);
+            result.AddRange(String.SerializeStrings());
+            result.AddRange([0x0F, 0xFF, 0xFF, 0xF1]);
+            result.AddRange(Ptr.SerializeFunctions());
+            result.AddRange([0x0F, 0xFF]);
+        }
+        return result.ToArray();
+    }
 }

Qrakhen.Qamp.Core/Values/Objects/Obj.cs

@@ -2,13 +2,16 @@
 
 public class Obj(ValueType type) : IValue
 {
+    public bool __gcMarked { get; private set; }
+    public int __gcCount { get; private set; } = 1;
+
     public readonly ValueType Type = type;
 
     public ValueType ValueType => Type;
 
     public static Value Create(Obj obj)
     {
-        return new Value(new Ptr(obj));
+        return new Value(Ptr.Create(obj));
     }
 
     public override string ToString() => "Obj<undefined>";

Qrakhen.Qamp.Core/Values/Objects/String.cs

@@ -1,13 +1,50 @@
-namespace Qrakhen.Qamp.Core.Values.Objects;
+using Qrakhen.Qamp.Core.Collections;
+using System.Text;
+
+namespace Qrakhen.Qamp.Core.Values.Objects;
 
 public class String(string? value) : Obj(ValueType.String)
 {
+    private static readonly Register<uint, String> _strings = new();
+
     public string? Value = value;
 
+    public override string ToString() => Value ?? "null";
+
+    public uint GetHash()
+    {
+        if (string.IsNullOrEmpty(value))
+            return 0;
+
+        return Value.GetHash();
+    }
+
     public static Value Make(string? value)
     {
-        return Create(new String(value));
+        String? str;
+        var hash = value?.GetHash() ?? 0;
+        if (!_strings.TryGet(hash, out str)) {
+            str = new String(value);
+            _strings.Add(hash, str);
         }
 
-    public override string ToString() => Value ?? "null";
+        return Create(str);
+    }
+
+    public static byte[] SerializeStrings()
+    {
+        List<byte> result = new();
+        foreach (var pair in _strings) { 
+            String str = pair.Value;
+            if (str == null || str.__gcMarked)
+                continue;
+            byte[] bytes;
+            bytes = Encoding.ASCII.GetBytes(str.Value!);
+            result.AddRange(pair.Key.GetBytes());
+            result.AddRange(bytes.Length.GetBytes());
+            result.AddRange(bytes);
+            result.Add(0);
+        }
+        return result.ToArray();
+    }
 }

Qrakhen.Qamp.Core/Values/Ptr.cs

@@ -2,13 +2,15 @@
 using System.Runtime.InteropServices;
 using Qrakhen.Qamp.Core.Collections;
 using System.Diagnostics.CodeAnalysis;
+using System.Security.Cryptography;
+using System.Text;
 
 namespace Qrakhen.Qamp.Core.Values;
 
 [StructLayout(LayoutKind.Sequential, Size = sizeof(ulong))]
 public readonly struct Ptr
 {
-    private static readonly PushStack<Obj> _register = new(0x10);
+    private static readonly PushStack<Obj> _register = new();
 
     [Serialized] public readonly Address Address;
 
@@ -26,10 +28,15 @@ public readonly struct Ptr
         }
     }
 
-    public Ptr(Obj obj)
+    private Ptr(Address address)
     {
         //_pointer = GCHandle.ToIntPtr(GCHandle.Alloc(obj, GCHandleType.Normal));
-        Address = _register.Add(obj);
+        Address = address;
+    }
+
+    public static Ptr Create(Obj obj)
+    {
+        return new Ptr(_register.Add(obj));
     }
 
     public T? As<T>(bool throwWhenNull = true) where T : Obj
@@ -41,4 +48,26 @@ public readonly struct Ptr
     }
 
     public override string ToString() => $"0x{Value}";
+
+    public static byte[] SerializeFunctions()
+    {
+        List<byte> result = new();
+        for (int i = 0; i < _register.Count; i++) {
+            Obj obj = _register.Get(i);
+            if (obj == null || obj.__gcMarked)
+                continue;
+            byte[] bytes;
+            if (obj is Function function) {
+                bytes = function.Serialize();
+            } else {
+                continue;
+            }
+
+            result.AddRange(i.GetBytes());
+            result.AddRange(bytes.Length.GetBytes());
+            result.AddRange(bytes);
+            result.Add(0);
+        }
+        return result.ToArray();
+    }
 }

Qrakhen.Qamp.Core/Values/Value.cs

@@ -12,7 +12,7 @@ public interface IValue
 }
 
 [StructLayout(LayoutKind.Explicit, Size = 0x10)]
-public readonly struct Value : IValue, IDebug<string>
+public readonly struct Value : IValue, ISerialize<Value>, IDebug<string>
 {
     public static readonly Value Void = new Value();
     public static readonly Value True = new Value(true);
@@ -140,6 +140,21 @@ public readonly struct Value : IValue, IDebug<string>
 
       return str + string.Join("\n  - ", lines);
     }
+
+    public byte[] Serialize()
+    {
+        List<byte> result = [.. ((ushort)Type).GetBytes() ];
+        byte[] bytes = Signed.GetBytes();
+        result.Add((byte)bytes.Length);
+        result.AddRange(bytes);
+        result.Add(0x7F);
+        return result.ToArray();
+    }
+
+    public static Value Deserialize(byte[] data)
+    {
+        return Void;
+    }
 }
 
 public class SerializedAttribute : Attribute;

README.md

@@ -42,6 +42,8 @@ Most basic Usage:
 # This is a comment.
 *~ q <~ 12;
 *~ f <~ (n) <: n < 2 ? n : f(n-1) + f(n-2);
+*~ a <~ [1, 2, 3];
+*~ x <~ a:0 + a:2; # = 4
 ```
 
 ### Assignments