From 7d794e8c8a181f2f5d470238b1e48923367a9be0 Mon Sep 17 00:00:00 2001 From: hoelzlc Date: Thu, 13 Feb 2025 16:39:35 +0100 Subject: [PATCH 1/3] first try on a preprocessor concept --- src/atomiq/language.py | 156 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 src/atomiq/language.py diff --git a/src/atomiq/language.py b/src/atomiq/language.py new file mode 100644 index 0000000..89e6f68 --- /dev/null +++ b/src/atomiq/language.py @@ -0,0 +1,156 @@ +from collections import namedtuple +from functools import wraps +import ast +from artiq.language import kernel as artiq_kernel +import inspect +from textwrap import dedent +import dis, builtins + +_ARTIQEmbeddedInfo = namedtuple("_ARTIQEmbeddedInfo", + "core_name portable function syscall forbidden destination flags") +def better_exec(code_, globals_=None, locals_=None, /, *, closure=None): + import ast + import linecache + + if not hasattr(better_exec, "saved_sources"): + old_getlines = linecache.getlines + better_exec.saved_sources = [] + + def patched_getlines(filename, module_globals=None): + if "")[0]) + return better_exec.saved_sources[index].splitlines(True) + else: + return old_getlines(filename, module_globals) + + linecache.getlines = patched_getlines + better_exec.saved_sources.append(code_) + exec( + compile( + ast.parse(code_), + filename=f"", + mode="exec", + ), + globals_, + locals_, + closure=None, + ) + + + + +class RealtimeTransformer(ast.NodeTransformer): + + def __init__(self, instance, globals_): + self.instance = instance + self.globals_ = globals_ + super().__init__() + + def visit_FunctionDef(self, node): + node.name = f"{node.name}__artiq" + i_realtime_dec = None + for i, decorator in enumerate(node.decorator_list): + if decorator.id == "realtime": + i_realtime_dec = i + if i_realtime_dec is not None: + del node.decorator_list[i_realtime_dec] + self.generic_visit(node) + return node + + def visit_Call(self, node): + func = None + if type(node.func) == ast.Name: + if node.func.id not in dir(builtins): + func = self.globals_[node.func.id] + + elif type(node.func) == ast.Attribute: + func = self.instance + func_node = node.func + attrs = [] + while isinstance(func_node, ast.Attribute): + attrs.append(func_node.attr) + func_node = func_node.value + for attr in reversed(attrs): + func = getattr(func, attr) + + if func is not None: + lines = inspect.getsource(func) + lines = dedent(lines) + t = ast.parse(lines) + #check for decorators here and proceed accordingly + + self.generic_visit(node) + return node + +class RealtimeEntryTransformer(RealtimeTransformer): + pass + + + + +class _Realtime: + flags = {} + instance = None + def __init__(self, fn, flags=None): + if flags is not None: + self.flags = flags + self.fn = fn + + def is_entry_point(self): + frame = inspect.getouterframes(inspect.currentframe())[2][0] + return type(frame.f_back.f_locals["self"]) != _Realtime + + + def __set_name__(self, owner, name): + self.owner = owner + self.name = name + + def __get__(self,instance, owner): + self.instance = instance + return self.__call__ + + def __call__(self, *args, **kwargs): + + print(inspect.stack()[1]) + self.fn.class_name = self.owner.__name__ + + lines = inspect.getsource(self.fn) + lines = dedent(lines) + fun_ast = ast.parse(lines) + loc = locals() + transformer = RealtimeTransformer(self.instance,self.fn.__globals__) + t = transformer.visit(fun_ast) + better_exec(ast.unparse(t), self.fn.__globals__, loc) + print("globals",self.fn.__globals__) + print("func",self.fn) + f = loc[f"{self.fn.__name__}__artiq"] + + self.f = artiq_kernel(f, self.flags) + + setattr(self.owner, f"{self.name}__artiq", self.f) + + if self.is_entry_point(): + # transformer = RealtimeEntryTransformer(self.instance,self.fn.__globals__) + # better_exec(ast.unparse(transformer.visit(fun_ast)), self.fn.__globals__, loc) + lines_new = lines.splitlines()[1:2] + lines_new.append(" self.prechunk_host__artiq(points)") + better_exec("\n".join(lines_new), self.fn.__globals__, loc) + f_entry = loc[self.fn.__name__] + setattr(self.owner, self.name, f_entry) + return getattr(self.instance,self.name)(*args, **kwargs) + else: + setattr(self.owner, self.name, None) + + + +def realtime(fn=None, flags=None): + """ + Does not work in nested classes somehow + """ + def _realtime(fn): + return _Realtime(fn,flags) + if fn is None: + return _realtime + else: + return _Realtime(fn,flags) + -- GitLab From f85771e2f0d8bddaa98c933dcc17f632f59e8e1c Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 13 Feb 2025 19:00:57 +0100 Subject: [PATCH 2/3] working principle of realtime visitor --- src/atomiq/language.py | 63 ++++++++++++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 17 deletions(-) diff --git a/src/atomiq/language.py b/src/atomiq/language.py index 89e6f68..d18bd7a 100644 --- a/src/atomiq/language.py +++ b/src/atomiq/language.py @@ -51,7 +51,8 @@ class RealtimeTransformer(ast.NodeTransformer): i_realtime_dec = None for i, decorator in enumerate(node.decorator_list): if decorator.id == "realtime": - i_realtime_dec = i + decorator.id = "kernel" + # i_realtime_dec = i if i_realtime_dec is not None: del node.decorator_list[i_realtime_dec] self.generic_visit(node) @@ -73,10 +74,22 @@ class RealtimeTransformer(ast.NodeTransformer): for attr in reversed(attrs): func = getattr(func, attr) + if func is not None: lines = inspect.getsource(func) lines = dedent(lines) - t = ast.parse(lines) + call_def_node = ast.parse(lines).body[0] + + is_realtime_call = False + for decorator in call_def_node.decorator_list: + if decorator.id == "realtime": + is_realtime_call = True + + if is_realtime_call: + if type(node.func) == ast.Name: + node.func.id = f"{node.func.id}__artiq" + elif type(node.func) == ast.Attribute: + node.func.attr = f"{node.func.attr}__artiq" #check for decorators here and proceed accordingly self.generic_visit(node) @@ -91,13 +104,18 @@ class RealtimeEntryTransformer(RealtimeTransformer): class _Realtime: flags = {} instance = None + sub_functions_constructed = False + def __init__(self, fn, flags=None): if flags is not None: self.flags = flags self.fn = fn - def is_entry_point(self): - frame = inspect.getouterframes(inspect.currentframe())[2][0] + def is_entry_point(self, frame): + """ + This function checks if this is the first function decorated with @realtime or if it was already called by + a @realtime function + """ return type(frame.f_back.f_locals["self"]) != _Realtime @@ -105,15 +123,8 @@ class _Realtime: self.owner = owner self.name = name - def __get__(self,instance, owner): - self.instance = instance - return self.__call__ - - def __call__(self, *args, **kwargs): - - print(inspect.stack()[1]) + def contruct_sub_functions(self, frame): self.fn.class_name = self.owner.__name__ - lines = inspect.getsource(self.fn) lines = dedent(lines) fun_ast = ast.parse(lines) @@ -121,22 +132,40 @@ class _Realtime: transformer = RealtimeTransformer(self.instance,self.fn.__globals__) t = transformer.visit(fun_ast) better_exec(ast.unparse(t), self.fn.__globals__, loc) - print("globals",self.fn.__globals__) - print("func",self.fn) f = loc[f"{self.fn.__name__}__artiq"] - self.f = artiq_kernel(f, self.flags) + # self.f = artiq_kernel(f, self.flags) - setattr(self.owner, f"{self.name}__artiq", self.f) + setattr(self.owner, f"{self.name}__artiq", f) - if self.is_entry_point(): + + if self.is_entry_point(frame): # transformer = RealtimeEntryTransformer(self.instance,self.fn.__globals__) # better_exec(ast.unparse(transformer.visit(fun_ast)), self.fn.__globals__, loc) lines_new = lines.splitlines()[1:2] + lines_new.append(" print(inspect.getsource(self.prechunk_host__artiq))") lines_new.append(" self.prechunk_host__artiq(points)") better_exec("\n".join(lines_new), self.fn.__globals__, loc) f_entry = loc[self.fn.__name__] setattr(self.owner, self.name, f_entry) + self.sub_functions_constructed = True + + def __get__(self,instance, owner): + self.instance = instance + frame = inspect.getouterframes(inspect.currentframe())[2][0] + print("__get__") + if type(frame.f_back.f_locals["self"]) == RealtimeTransformer: + #todo: make this more robust + return self.fn + if not self.sub_functions_constructed: + self.contruct_sub_functions(frame) + + return self.__call__ + + def __call__(self, *args, **kwargs): + print("__call__", self.instance) + frame = inspect.getouterframes(inspect.currentframe())[2][0] + if self.is_entry_point(frame): return getattr(self.instance,self.name)(*args, **kwargs) else: setattr(self.owner, self.name, None) -- GitLab From 179b39f07c8ad2cee6fe86e919fa5e0223026550 Mon Sep 17 00:00:00 2001 From: Christian Date: Fri, 14 Feb 2025 15:00:56 +0100 Subject: [PATCH 3/3] small proof of principle --- src/atomiq/language.py | 111 +++++++++++++++++++++++++---------------- 1 file changed, 68 insertions(+), 43 deletions(-) diff --git a/src/atomiq/language.py b/src/atomiq/language.py index d18bd7a..e98b285 100644 --- a/src/atomiq/language.py +++ b/src/atomiq/language.py @@ -8,27 +8,27 @@ import dis, builtins _ARTIQEmbeddedInfo = namedtuple("_ARTIQEmbeddedInfo", "core_name portable function syscall forbidden destination flags") -def better_exec(code_, globals_=None, locals_=None, /, *, closure=None): +def inspectable_exec(code_, globals_=None, locals_=None, /, *, closure=None): import ast import linecache - if not hasattr(better_exec, "saved_sources"): + if not hasattr(inspectable_exec, "saved_sources"): old_getlines = linecache.getlines - better_exec.saved_sources = [] + inspectable_exec.saved_sources = [] def patched_getlines(filename, module_globals=None): if "")[0]) - return better_exec.saved_sources[index].splitlines(True) + return inspectable_exec.saved_sources[index].splitlines(True) else: return old_getlines(filename, module_globals) linecache.getlines = patched_getlines - better_exec.saved_sources.append(code_) + inspectable_exec.saved_sources.append(code_) exec( compile( ast.parse(code_), - filename=f"", + filename=f"", mode="exec", ), globals_, @@ -41,18 +41,27 @@ def better_exec(code_, globals_=None, locals_=None, /, *, closure=None): class RealtimeTransformer(ast.NodeTransformer): - def __init__(self, instance, globals_): + def __init__(self, realtime_name, instance, globals_, replacement_decorator, skip_functions): self.instance = instance self.globals_ = globals_ + self.realtime_name = realtime_name + self.replacement_decorator = replacement_decorator + self.skip_functions = skip_functions super().__init__() + def visit_Expr(self, node): + super().generic_visit(node) + if hasattr(node,"value"): + return node def visit_FunctionDef(self, node): - node.name = f"{node.name}__artiq" + node.name = f"{node.name}__{self.realtime_name}" i_realtime_dec = None for i, decorator in enumerate(node.decorator_list): if decorator.id == "realtime": - decorator.id = "kernel" - # i_realtime_dec = i + if self.replacement_decorator is not None: + decorator.id = self.replacement_decorator + else: + i_realtime_dec = i if i_realtime_dec is not None: del node.decorator_list[i_realtime_dec] self.generic_visit(node) @@ -65,13 +74,18 @@ class RealtimeTransformer(ast.NodeTransformer): func = self.globals_[node.func.id] elif type(node.func) == ast.Attribute: - func = self.instance + print(ast.dump(node)) func_node = node.func attrs = [] while isinstance(func_node, ast.Attribute): attrs.append(func_node.attr) func_node = func_node.value + if func_node.id == "self": + func = self.instance + else: + func = self.globals_[func_node.id] for attr in reversed(attrs): + #this does not handle functions which are defined within the function, we need to process them also before func = getattr(func, attr) @@ -81,26 +95,29 @@ class RealtimeTransformer(ast.NodeTransformer): call_def_node = ast.parse(lines).body[0] is_realtime_call = False + if call_def_node.name in self.skip_functions: + #make this handle functions more precisely not only by name, require full module path + return None for decorator in call_def_node.decorator_list: - if decorator.id == "realtime": - is_realtime_call = True + try: + # this here does not work with async functions yet, ignores them for now + if decorator.id == "realtime": + is_realtime_call = True + if decorator.id in self.skip_functions: + return None + except AttributeError: + pass + # This needs to expanded to also capture explicit calls without decorators if is_realtime_call: if type(node.func) == ast.Name: - node.func.id = f"{node.func.id}__artiq" + node.func.id = f"{node.func.id}__{self.realtime_name}" elif type(node.func) == ast.Attribute: - node.func.attr = f"{node.func.attr}__artiq" - #check for decorators here and proceed accordingly + node.func.attr = f"{node.func.attr}__{self.realtime_name}" self.generic_visit(node) return node -class RealtimeEntryTransformer(RealtimeTransformer): - pass - - - - class _Realtime: flags = {} instance = None @@ -116,7 +133,8 @@ class _Realtime: This function checks if this is the first function decorated with @realtime or if it was already called by a @realtime function """ - return type(frame.f_back.f_locals["self"]) != _Realtime + #todo: make more robust + return type(frame.f_back.f_locals["self"]) not in [_Realtime, RealtimeTransformer] def __set_name__(self, owner, name): @@ -124,28 +142,36 @@ class _Realtime: self.name = name def contruct_sub_functions(self, frame): + self.fn.class_name = self.owner.__name__ lines = inspect.getsource(self.fn) lines = dedent(lines) - fun_ast = ast.parse(lines) loc = locals() - transformer = RealtimeTransformer(self.instance,self.fn.__globals__) - t = transformer.visit(fun_ast) - better_exec(ast.unparse(t), self.fn.__globals__, loc) - f = loc[f"{self.fn.__name__}__artiq"] - # self.f = artiq_kernel(f, self.flags) + # here we could make a loop over all registered real time devices + artiq_transformer = RealtimeTransformer("artiq",self.instance,self.fn.__globals__,"kernel",[]) + artiq_ast = artiq_transformer.visit(ast.parse(lines)) + inspectable_exec(ast.unparse(artiq_ast), self.fn.__globals__, loc) + f_artiq = loc[f"{self.fn.__name__}__artiq"] + + host_transformer = RealtimeTransformer("host",self.instance,self.fn.__globals__,None,["kernel","delay"]) + host_ast = host_transformer.visit(ast.parse(lines)) + inspectable_exec(ast.unparse(host_ast), self.fn.__globals__, loc) + f_host = loc[f"{self.fn.__name__}__host"] - setattr(self.owner, f"{self.name}__artiq", f) + setattr(self.owner, f"{self.name}__artiq", f_artiq) + setattr(self.owner, f"{self.name}__host", f_host) if self.is_entry_point(frame): - # transformer = RealtimeEntryTransformer(self.instance,self.fn.__globals__) - # better_exec(ast.unparse(transformer.visit(fun_ast)), self.fn.__globals__, loc) + #this part is hardcoded as a demonstration, must be implemented using the ast lines_new = lines.splitlines()[1:2] - lines_new.append(" print(inspect.getsource(self.prechunk_host__artiq))") + #running the host part first also allows to implement some precalculation decorator + lines_new.append(" self.log.info('host part:' + inspect.getsource(self.prechunk_host__host))") + lines_new.append(" self.prechunk_host__host(points)") + lines_new.append(" self.log.info('kernel part:'+ inspect.getsource(self.prechunk_host__artiq))") lines_new.append(" self.prechunk_host__artiq(points)") - better_exec("\n".join(lines_new), self.fn.__globals__, loc) + inspectable_exec("\n".join(lines_new), self.fn.__globals__, loc) f_entry = loc[self.fn.__name__] setattr(self.owner, self.name, f_entry) self.sub_functions_constructed = True @@ -153,22 +179,21 @@ class _Realtime: def __get__(self,instance, owner): self.instance = instance frame = inspect.getouterframes(inspect.currentframe())[2][0] - print("__get__") + if not self.sub_functions_constructed: + self.contruct_sub_functions(frame) + if type(frame.f_back.f_locals["self"]) == RealtimeTransformer: #todo: make this more robust return self.fn - if not self.sub_functions_constructed: - self.contruct_sub_functions(frame) return self.__call__ def __call__(self, *args, **kwargs): - print("__call__", self.instance) - frame = inspect.getouterframes(inspect.currentframe())[2][0] - if self.is_entry_point(frame): - return getattr(self.instance,self.name)(*args, **kwargs) - else: - setattr(self.owner, self.name, None) + # frame = inspect.getouterframes(inspect.currentframe())[2][0] + # if self.is_entry_point(frame): + return getattr(self.instance,self.name)(*args, **kwargs) + # else: + # setattr(self.owner, self.name, None) -- GitLab