From 70f585f5fc60535fd3743064a93972d8b2327f1d Mon Sep 17 00:00:00 2001 From: rarescodemart Date: Fri, 25 Nov 2022 14:32:46 +0200 Subject: [PATCH] WID-100: add functionality to display the description of the component arguments --- .gitignore | 5 + src/components/CustomNodeModel.ts | 8 +- src/components/CustomNodeWidget.tsx | 99 ++++++-- src/components/port/CustomPortLabel.tsx | 16 +- src/components/port/CustomPortModel.ts | 11 + src/components/port/WithToggle.tsx | 50 ++++ src/components/port/types.ts | 14 ++ src/tray_library/AdvanceComponentLib.tsx | 20 +- style/ComponentInfo.css | 5 +- style/base.css | 13 + .../kerastransferlearningmodel.json | 16 ++ xircuits/handlers/component_parser.py | 236 ++++++++++++++++++ xircuits/handlers/components.py | 223 +---------------- 13 files changed, 468 insertions(+), 248 deletions(-) create mode 100644 src/components/port/WithToggle.tsx create mode 100644 src/components/port/types.ts create mode 100644 xai_components/xai_tensorflow_keras/description/kerastransferlearningmodel.json create mode 100644 xircuits/handlers/component_parser.py diff --git a/.gitignore b/.gitignore index 223c36fb..9cfd3bd2 100644 --- a/.gitignore +++ b/.gitignore @@ -143,3 +143,8 @@ tsconfig.tsbuildinfo # .py files compiled by Xircuits examples/*.py + +# pycharm files +/.idea/ + +xai_components/*/arguments \ No newline at end of file diff --git a/src/components/CustomNodeModel.ts b/src/components/CustomNodeModel.ts index 6d9b9281..b4a05ef0 100644 --- a/src/components/CustomNodeModel.ts +++ b/src/components/CustomNodeModel.ts @@ -41,11 +41,13 @@ export class CustomNodeModel extends DefaultNodeModel { this.extras=event.data.extras; } - addOutPortEnhance(label: string, name: string, after: boolean = true, id?: string): CustomPortModel { + addOutPortEnhance(label: string, name: string, after: boolean = true, id?: string, description: string = ""): CustomPortModel { //check if portID is passed, if not SR will generate a new port ID const p = (id) ? new CustomPortModel({in: false, name: name, label: label, id:id}) : new CustomPortModel({in: false, name: name, label: label}); + + p.description = description; if (!after) { this.portsOut.splice(0, 0, p); @@ -54,12 +56,14 @@ export class CustomNodeModel extends DefaultNodeModel { return this.addPort(p); } - addInPortEnhance(label: string, name: string, after: boolean = true, id?: string): CustomPortModel { + addInPortEnhance(label: string, name: string, after: boolean = true, id?: string, description: string = ""): CustomPortModel { //check if portID is passed, if not SR will generate a new port ID const p = (id) ? new CustomPortModel({in: true, name: name, label: label, id:id}) : new CustomPortModel({in: true, name: name, label: label}); + p.description = description; + if (!after) { this.portsOut.splice(0, 0, p); } diff --git a/src/components/CustomNodeWidget.tsx b/src/components/CustomNodeWidget.tsx index ed2b17f5..c9ef875f 100644 --- a/src/components/CustomNodeWidget.tsx +++ b/src/components/CustomNodeWidget.tsx @@ -102,9 +102,10 @@ export interface DefaultNodeProps { */ export class CustomNodeWidget extends React.Component { - generatePort = (port) => { - return ; - }; + portsNo = this.props.node.getInPorts().length + this.props.node.getOutPorts().length; + + tooltipDescriptionRef = React.createRef(); + element:Object; state = { @@ -127,7 +128,63 @@ export class CustomNodeWidget extends React.Component { original: 'https://picsum.photos/id/1019/1000/600/', thumbnail: 'https://picsum.photos/id/1019/250/150/' }, - ] + ], + showParamDescriptionList: new Array(this.portsNo).fill(false), + paramName: "" + }; + + /** + * creates a particular function for each component so that it can set only it's state + * @param id + */ + setShowParamDescription = (id : number) => { + const _setShowParamDescription = (newShowDescription : boolean) => { + this.setState({ + showParamDescriptionList: this.state.showParamDescriptionList.map((value, index) => ( + id === index ? newShowDescription : false + ) + ), + showDescription: false + }) + } + return _setShowParamDescription; + } + + setDescriptionStr = (paramName: string) => { + const _setDescriptionStr = async (descriptionStr : string) => { + await this.setState({ + descriptionStr : descriptionStr, + paramName: paramName + }); + ReactTooltip.show(this.element as Element); + } + return _setDescriptionStr; + } + + generatePort = (port, index) => { + const argumentDescriptions = this.props.node['extras']['argumentDescriptions']; + + // remove the ☆ from the beginning of the label + const name = port.getOptions().label[0] === "★" ? port.getOptions().label.slice(1) : port.getOptions().label; + + const description = argumentDescriptions && (name in argumentDescriptions) ? argumentDescriptions[name] : ""; + + const isOutPort = port.getOptions().name.includes('parameter-out'); + + index = isOutPort ? index + this.props.node.getInPorts().length: index; + + return ( + + ); }; showTooltip() { @@ -206,8 +263,12 @@ export class CustomNodeWidget extends React.Component { * Show/Hide Component's Description Tooltip */ async handleDescription() { - await this.setState({ showDescription: !this.state.showDescription }); - this.getDescriptionStr(); + await this.setState({ + showDescription: !this.state.showDescription, + showParamDescriptionList: new Array(this.portsNo).fill(false), + paramName: "" + }); + await this.getDescriptionStr(); ReactTooltip.show(this.element as Element); } @@ -234,7 +295,7 @@ export class CustomNodeWidget extends React.Component { delete this.props.node.getOptions().extras["tip"]; this.props.node.getOptions().extras["borderColor"]="rgb(0,192,255)"; } - + render() { if (this.props.node['extras']['type'] == 'comment') { return ( @@ -301,39 +362,43 @@ export class CustomNodeWidget extends React.Component { {/** Description Tooltip */} - {this.state.showDescription && prev || cur, false)) && { this.setState({ showDescription: true }) }} - afterHide={() => { this.setState({ showDescription: false }) }} delayHide={60000} - delayUpdate={5000} + delayUpdate={0} getContent={() => -
+
- {this.props.node.getOptions()["name"]} + {this.props.node.getOptions()["name"] + " " + this.state.paramName}

Description:

-
e.stopPropagation()} className='description-container'> -
+
} overridePosition={( { left, top }, currentEvent, currentTarget, node, refNode) => { + const currentNode = this.props.node; const nodeDimension = { x: currentNode.width, y: currentNode.height }; const nodePosition = { x: currentNode.getX(), y: currentNode.getY() }; + let newPositionX = nodePosition.x; let newPositionY = nodePosition.y; let offset = 0; @@ -346,7 +411,7 @@ export class CustomNodeWidget extends React.Component { if (refNode == 'top') { newPositionX = newPositionX - 208 + offset + (nodeDimension.x / 2); - newPositionY = newPositionY - 220; + newPositionY = newPositionY + 66 - this.tooltipDescriptionRef.current.clientHeight; } else if (refNode == 'bottom') { newPositionX = newPositionX - 208 + offset + (nodeDimension.x / 2); diff --git a/src/components/port/CustomPortLabel.tsx b/src/components/port/CustomPortLabel.tsx index 208293ed..3b0d1ac8 100644 --- a/src/components/port/CustomPortLabel.tsx +++ b/src/components/port/CustomPortLabel.tsx @@ -2,11 +2,17 @@ import * as React from 'react'; import { DiagramEngine, PortWidget } from '@projectstorm/react-diagrams-core'; import { DefaultNodeModel, DefaultPortModel } from "@projectstorm/react-diagrams"; import styled from '@emotion/styled'; +import WithToggle from "./WithToggle"; + export interface CustomPortLabelProps { port: DefaultPortModel; engine: DiagramEngine; node: DefaultNodeModel; + showDescription: boolean; + setShowDescription: any; + setDescriptionStr: (string: any) => any; + description : string; } namespace S { @@ -123,7 +129,15 @@ export class CustomPortLabel extends React.Component { const label = ( - {this.props.port.getOptions().label} + + {this.props.port.getOptions().label} + ); return ( diff --git a/src/components/port/CustomPortModel.ts b/src/components/port/CustomPortModel.ts index 4c2076dd..e3a1a919 100644 --- a/src/components/port/CustomPortModel.ts +++ b/src/components/port/CustomPortModel.ts @@ -8,6 +8,17 @@ import {PortModel} from "@projectstorm/react-diagrams-core"; */ export class CustomPortModel extends DefaultPortModel { + private _description : string; + + get description(): string { + return this._description; + } + + set description(value: string) { + this._description = value; + } + + canLinkToPort(port: PortModel): boolean { if (port instanceof DefaultPortModel) { diff --git a/src/components/port/WithToggle.tsx b/src/components/port/WithToggle.tsx new file mode 100644 index 00000000..7766b8fe --- /dev/null +++ b/src/components/port/WithToggle.tsx @@ -0,0 +1,50 @@ +import * as React from "react"; +import Toggle from 'react-toggle' +import {useCallback, useRef} from "react"; +import {WithToggleProps} from "./types"; + + +export default function WithToggle(props: WithToggleProps){ + const ref = useRef(null); + + const changeHandler = useCallback(async () => { + await props.setShowDescription(!props.showDescription) + await props.setDescriptionStr(props.description) + }, [props.description, props.showDescription]) + + return ( +
+ {props.renderToggleBeforeChildren ? + <> + { + props.description && + + } + + {props.children} + + + : + <> + + {props.children} + + { + props.description && + + } + + } +
+ ) +} \ No newline at end of file diff --git a/src/components/port/types.ts b/src/components/port/types.ts new file mode 100644 index 00000000..6b9ccc02 --- /dev/null +++ b/src/components/port/types.ts @@ -0,0 +1,14 @@ +import {ReactElement} from "react"; + +export type WithToggleProps = { + renderToggleBeforeChildren : boolean; + children: ReactElement[] | ReactElement | string; + showDescription: boolean; + setShowDescription: any; + setDescriptionStr: (param: string) => void; + description: string; +} + +export type WithToggleState = { + showDescription : boolean +} \ No newline at end of file diff --git a/src/tray_library/AdvanceComponentLib.tsx b/src/tray_library/AdvanceComponentLib.tsx index 325b08ca..753f2261 100644 --- a/src/tray_library/AdvanceComponentLib.tsx +++ b/src/tray_library/AdvanceComponentLib.tsx @@ -31,7 +31,8 @@ export function AdvancedComponentLibrary(props: AdvancedComponentLibraryProps) { extras: { "type": nodeData.type, "path": nodeData.file_path, - "description": nodeData.docstring, + "description": nodeData["json_description"]["description"] || nodeData.docstring, + "argumentDescriptions" : nodeData["json_description"]["arguments"], "lineNo": nodeData.lineno } }); @@ -44,19 +45,24 @@ export function AdvancedComponentLibrary(props: AdvancedComponentLibraryProps) { "str": "string" } - nodeData["variables"].forEach(variable => { - let name = variable["name"]; - let type = type_name_remappings[variable["type"]] || variable["type"]; + const argumentDescriptions = nodeData["json_description"]["arguments"]; + + + nodeData["variables"].forEach((variable, _) => { + const name = variable["name"]; + const type = type_name_remappings[variable["type"]] || variable["type"]; + + const description = argumentDescriptions ? argumentDescriptions[name] || "" : ""; switch (variable["kind"]) { case "InCompArg": - node.addInPortEnhance(`★${name}`, `parameter-${type}-${name}`); + node.addInPortEnhance(`★${name}`, `parameter-${type}-${name}`, true, null, description); break; case "InArg": - node.addInPortEnhance(name, `parameter-${type}-${name}`); + node.addInPortEnhance(name, `parameter-${type}-${name}`, true, null, description); break; case "OutArg": - node.addOutPortEnhance(name, `parameter-out-${type}-${name}`); + node.addOutPortEnhance(name, `parameter-out-${type}-${name}`, true, null, description); break; case "BaseComponent": node.addOutPortEnhance(`${name} ▶`, `out-flow-${name}`); diff --git a/style/ComponentInfo.css b/style/ComponentInfo.css index 7d750d0d..dd6d4eb9 100644 --- a/style/ComponentInfo.css +++ b/style/ComponentInfo.css @@ -7,9 +7,10 @@ } .description-container{ - width: 450px; - height: 250px; + width: 600px; + max-height: 250px; overflow: auto; + display: inline-block; } .description-title{ diff --git a/style/base.css b/style/base.css index 94b273d4..f0b6d12b 100644 --- a/style/base.css +++ b/style/base.css @@ -54,4 +54,17 @@ input[type=number]::-webkit-inner-spin-button { /* Fix arrow position of HTMLSelect tag on run dialog */ .jp-Dialog-content .f1ozlkqi { position: relative; +} + +.alignToggle { + display: flex; + flex-direction: row; + align-items: center; + justify-content: flex-start; +} + +.alignToggle span { + display: inline-block; + padding: 0; + margin: 0; } \ No newline at end of file diff --git a/xai_components/xai_tensorflow_keras/description/kerastransferlearningmodel.json b/xai_components/xai_tensorflow_keras/description/kerastransferlearningmodel.json new file mode 100644 index 00000000..3a55f042 --- /dev/null +++ b/xai_components/xai_tensorflow_keras/description/kerastransferlearningmodel.json @@ -0,0 +1,16 @@ +{ + "description" : "keras_transfer_learning description :math:`1/\\tau_{0}`", + "arguments" : { + "base_model_name" : "base_model_name description", + "include_top" : "include_top description :math:`1/\\tau_{0}`", + "weights" : "weights description", + "input_shape" : "input_shape description", + "freeze_all" : "freeze_all description", + "fine_tune_from" : "fine_tune_from description", + "classes" : "classes description", + "binary" : "binary description", + "classifier_activation" : "classifier_activation description", + "kwargs" : "kwargs description", + "model" : "model description" + } +} \ No newline at end of file diff --git a/xircuits/handlers/component_parser.py b/xircuits/handlers/component_parser.py new file mode 100644 index 00000000..25794ff4 --- /dev/null +++ b/xircuits/handlers/component_parser.py @@ -0,0 +1,236 @@ +import ast +import json +import os +import pathlib +import sys +import traceback +from itertools import chain +import platform + +from .config import get_config + +DEFAULT_COMPONENTS_PATHS = [ + os.path.join(os.path.dirname(__file__), "..", "..", "xai_components"), + "xai_components", + os.path.expanduser("~/xai_components"), + os.environ.get("XPIPES_COMPONENTS_DIR") +] + +# Get the default components from here for now +# A better place may be a config file, or turning them into real components +# A good point in time to do that, would be when the python compilation step +# gets refactored +DEFAULT_COMPONENTS = { + 1: { "name": "Get Argument String Name", "returnType": "string","color":"lightpink"}, + 2: { "name": "Get Argument Integer Name", "returnType": "int","color":"blue"}, + 3: { "name": "Get Argument Float Name", "returnType": "float","color":"green"}, + 4: { "name": "Get Argument Boolean Name", "returnType": "boolean","color":"red"}, + 5: { "name": "Literal String", "returnType": "string","color":"lightpink"}, + 6:{ "name": "Literal Integer", "returnType": "int","color":"blue"}, + 7:{ "name": "Literal Float", "returnType": "float","color":"green"}, + 8:{ "name": "Literal True", "returnType": "boolean","color":"red"}, + 9:{ "name": "Literal False", "returnType": "boolean","color":"red"}, + 10:{ "name": "Literal List", "returnType": "list","color":"yellow"}, + 11:{ "name": "Literal Tuple", "returnType": "tuple","color":"purple"}, + 12:{ "name": "Literal Dict", "returnType": "dict","color":"orange"}, + # Comment this first since we don't use it + # 1: { "name": "Math Operation", "returnType": "math"}, + # 2: { "name": "Convert to Aurora", "returnType": "convert"}, + # 7: { "name": "Debug Image", "returnType": "debug"}, + # 8: { "name": "Reached Target Accuracy", "returnType": "enough"}, +} + +COLOR_PALETTE = [ + "rgb(192,255,0)", + "rgb(0,102,204)", + "rgb(255,153,102)", + "rgb(255,102,102)", + "rgb(15,255,255)", + "rgb(255,204,204)", + "rgb(153,204,51)", + "rgb(255,153,0)", + "rgb(255,204,0)", + "rgb(204,204,204)", + "rgb(153,204,204)", + "rgb(153,0,102)", + "rgb(102,51,102)", + "rgb(153,51,204)", + "rgb(102,102,102)", + "rgb(255,102,0)", + "rgb(51,51,51)" +] + +GROUP_GENERAL = "GENERAL" +GROUP_ADVANCED = "ADVANCED" + +def remove_prefix(input_str, prefix): + prefix_len = len(prefix) + if input_str[0:prefix_len] == prefix: + return input_str[prefix_len:] + else: + return input_str + +def read_orig_code(node: ast.AST, lines): + line_from = node.lineno - 1 + col_from = node.col_offset + + line_to = node.end_lineno - 1 + col_to = node.end_col_offset + + if line_from == line_to: + line = lines[line_from] + return line[col_from:col_to] + else: + start_line = lines[line_from][col_from:] + between_lines = lines[(line_from+1):line_to] + end_line = lines[line_to][col_to] + return "\n".join(chain([start_line], between_lines, [end_line])) + + + +class ComponentsParser: + + def get_components(self): + components = [] + + for id, c in DEFAULT_COMPONENTS.items(): + components.append({ + "task": c["name"], + "header": GROUP_GENERAL, + "category": GROUP_GENERAL, + "variables": [], + "type": c["returnType"], + "color": c.get('color') or None + }) + + default_paths = set(pathlib.Path(p).expanduser().resolve() for p in sys.path) + + visited_directories = [] + for directory_string in self.get_component_directories(): + if directory_string is not None: + directory = pathlib.Path(directory_string).absolute() + if directory.exists() \ + and directory.is_dir() \ + and not any(pathlib.Path.samefile(directory, d) for d in visited_directories): + visited_directories.append(directory) + python_files = directory.rglob("xai_*/*.py") + + python_path = directory.expanduser().resolve() + + if python_path.parent in default_paths: + python_path = None + + try: + components.extend(chain.from_iterable(self.extract_components(f, directory, python_path) for f in python_files)) + except Exception: + error_msg = traceback.format_exc() + pass + finally: + components.extend(chain.from_iterable(self.extract_components(f, directory, python_path) for f in python_files)) + + components = list({(c["header"], c["task"]): c for c in components}.values()) + return components + + def get(self): + error_msg = "" + + components = self.get_components() + + # Set up component colors according to palette + for idx, c in enumerate(components): + if c.get("color") is None: + c["color"] = COLOR_PALETTE[idx % len(COLOR_PALETTE)] + + data = {"components": components, + "error_msg": error_msg} + return json.dumps(data) + + def get_component_directories(self): + paths = list(DEFAULT_COMPONENTS_PATHS) + paths.append(get_config().get("DEV", "BASE_PATH")) + return paths + + def extract_components(self, file_path, base_dir, python_path): + with open(file_path) as f: + lines = f.readlines() + + parse_tree = ast.parse(file_path.read_text(), file_path) + # Look for top level class definitions that are decorated with "@xai_component" + is_xai_component = lambda node: isinstance(node, ast.ClassDef) and \ + any((isinstance(decorator, ast.Call) and decorator.func.id == "xai_component") or \ + (isinstance(decorator, ast.Name) and decorator.id == "xai_component") + for decorator in node.decorator_list) + + return [self.extract_component(node, file_path.relative_to(base_dir), lines, python_path) + for node in parse_tree.body if is_xai_component(node)] + + def extract_component(self, node: ast.ClassDef, file_path, file_lines, python_path): + name = node.name + + keywords = {kw.arg: kw.value.value for kw in chain.from_iterable(decorator.keywords + for decorator in node.decorator_list + if isinstance(decorator, ast.Call) and decorator.func.id == "xai_component")} + + # Group Name for Display + category = remove_prefix(file_path.parent.name, "xai_").upper() + + is_arg = lambda n: isinstance(n, ast.AnnAssign) and \ + isinstance(n.annotation, ast.Subscript) and \ + n.annotation.value.id in ['InArg', 'InCompArg', 'OutArg'] + + is_flow_arg = lambda n: isinstance(n, ast.AnnAssign) and \ + isinstance(n.annotation, ast.Name) and \ + n.annotation.id in ['BaseComponent'] + + python_version = platform.python_version_tuple() + + variables = [] + for v in (node.body): + if is_flow_arg(v): + variables.append({ + "name": v.target.id, + "kind": v.annotation.id, + }) + continue + elif is_arg(v): + variables.append({ + "name": v.target.id, + "kind": v.annotation.value.id, + "type": read_orig_code(v.annotation.slice.value if int(python_version[1]) == 8 else v.annotation.slice, file_lines) + }) + continue + + docstring = ast.get_docstring(node) + lineno = [ + { + "lineno": node.lineno, + "end_lineno": node.end_lineno + } + ] + + + description = {} + path = os.path.join("xai_components", os.path.dirname(file_path), "description", str(node.name).lower() + ".json") + if os.path.isfile(path): + with open(path) as file: + description = json.load(file) + + + output = { + "class": name, + "package_name": ("xai_components." if python_path is None else "") + file_path.as_posix().replace("/", ".")[:-3], + "python_path": str(python_path) if python_path is not None else None, + "abs_file_path": os.path.join(str(python_path), str(file_path)) if python_path is not None else None, + "file_path": "xai_components/" + (file_path.as_posix()[:-3] + ".py" if platform.system() == "Windows" else str(file_path)), + "task": name, + "header": GROUP_ADVANCED, + "category": category, + "type": "debug", + "variables": variables, + "json_description": description, + "docstring": docstring, + "lineno" : lineno + } + output.update(keywords) + + return output diff --git a/xircuits/handlers/components.py b/xircuits/handlers/components.py index 876ef2f5..ede5af75 100644 --- a/xircuits/handlers/components.py +++ b/xircuits/handlers/components.py @@ -1,226 +1,11 @@ -import json -import os -import pathlib -import sys -import ast -from itertools import chain -import traceback - import tornado from jupyter_server.base.handlers import APIHandler -import platform - -from .config import get_config - -DEFAULT_COMPONENTS_PATHS = [ - os.path.join(os.path.dirname(__file__), "..", "..", "xai_components"), - "xai_components", - os.path.expanduser("~/xai_components"), - os.environ.get("XPIPES_COMPONENTS_DIR") -] - -# Get the default components from here for now -# A better place may be a config file, or turning them into real components -# A good point in time to do that, would be when the python compilation step -# gets refactored -DEFAULT_COMPONENTS = { - 1: { "name": "Get Argument String Name", "returnType": "string","color":"lightpink"}, - 2: { "name": "Get Argument Integer Name", "returnType": "int","color":"blue"}, - 3: { "name": "Get Argument Float Name", "returnType": "float","color":"green"}, - 4: { "name": "Get Argument Boolean Name", "returnType": "boolean","color":"red"}, - 5: { "name": "Literal String", "returnType": "string","color":"lightpink"}, - 6:{ "name": "Literal Integer", "returnType": "int","color":"blue"}, - 7:{ "name": "Literal Float", "returnType": "float","color":"green"}, - 8:{ "name": "Literal True", "returnType": "boolean","color":"red"}, - 9:{ "name": "Literal False", "returnType": "boolean","color":"red"}, - 10:{ "name": "Literal List", "returnType": "list","color":"yellow"}, - 11:{ "name": "Literal Tuple", "returnType": "tuple","color":"purple"}, - 12:{ "name": "Literal Dict", "returnType": "dict","color":"orange"}, - # Comment this first since we don't use it - # 1: { "name": "Math Operation", "returnType": "math"}, - # 2: { "name": "Convert to Aurora", "returnType": "convert"}, - # 7: { "name": "Debug Image", "returnType": "debug"}, - # 8: { "name": "Reached Target Accuracy", "returnType": "enough"}, -} - -COLOR_PALETTE = [ - "rgb(192,255,0)", - "rgb(0,102,204)", - "rgb(255,153,102)", - "rgb(255,102,102)", - "rgb(15,255,255)", - "rgb(255,204,204)", - "rgb(153,204,51)", - "rgb(255,153,0)", - "rgb(255,204,0)", - "rgb(204,204,204)", - "rgb(153,204,204)", - "rgb(153,0,102)", - "rgb(102,51,102)", - "rgb(153,51,204)", - "rgb(102,102,102)", - "rgb(255,102,0)", - "rgb(51,51,51)" -] - -GROUP_GENERAL = "GENERAL" -GROUP_ADVANCED = "ADVANCED" - -def remove_prefix(input_str, prefix): - prefix_len = len(prefix) - if input_str[0:prefix_len] == prefix: - return input_str[prefix_len:] - else: - return input_str - -def read_orig_code(node: ast.AST, lines): - line_from = node.lineno - 1 - col_from = node.col_offset - - line_to = node.end_lineno - 1 - col_to = node.end_col_offset - - if line_from == line_to: - line = lines[line_from] - return line[col_from:col_to] - else: - start_line = lines[line_from][col_from:] - between_lines = lines[(line_from+1):line_to] - end_line = lines[line_to][col_to] - return "\n".join(chain([start_line], between_lines, [end_line])) +from .component_parser import ComponentsParser class ComponentsRouteHandler(APIHandler): + + component_parser = ComponentsParser() @tornado.web.authenticated def get(self): - components = [] - error_msg = "" - - for id, c in DEFAULT_COMPONENTS.items(): - components.append({ - "task": c["name"], - "header": GROUP_GENERAL, - "category": GROUP_GENERAL, - "variables": [], - "type": c["returnType"], - "color":c.get('color') or None - }) - - default_paths = set(pathlib.Path(p).expanduser().resolve() for p in sys.path) - - visited_directories = [] - for directory_string in self.get_component_directories(): - if directory_string is not None: - directory = pathlib.Path(directory_string).absolute() - if directory.exists() \ - and directory.is_dir() \ - and not any(pathlib.Path.samefile(directory, d) for d in visited_directories): - visited_directories.append(directory) - python_files = directory.rglob("xai_*/*.py") - - python_path = directory.expanduser().resolve() - - if python_path.parent in default_paths: - python_path = None - - try: - components.extend(chain.from_iterable(self.extract_components(f, directory, python_path) for f in python_files)) - except Exception: - error_msg = traceback.format_exc() - pass - finally: - components.extend(chain.from_iterable(self.extract_components(f, directory, python_path) for f in python_files)) - - - components = list({(c["header"], c["task"]): c for c in components}.values()) - - # Set up component colors according to palette - for idx, c in enumerate(components): - if c.get("color") is None: - c["color"] = COLOR_PALETTE[idx % len(COLOR_PALETTE)] - - data = {"components": components, - "error_msg" : error_msg} - - self.finish(json.dumps(data)) - - def get_component_directories(self): - paths = list(DEFAULT_COMPONENTS_PATHS) - paths.append(get_config().get("DEV", "BASE_PATH")) - return paths - - def extract_components(self, file_path, base_dir, python_path): - with open(file_path) as f: - lines = f.readlines() - - parse_tree = ast.parse(file_path.read_text(), file_path) - # Look for top level class definitions that are decorated with "@xai_component" - is_xai_component = lambda node: isinstance(node, ast.ClassDef) and \ - any((isinstance(decorator, ast.Call) and decorator.func.id == "xai_component") or \ - (isinstance(decorator, ast.Name) and decorator.id == "xai_component") - for decorator in node.decorator_list) - - return [self.extract_component(node, file_path.relative_to(base_dir), lines, python_path) - for node in parse_tree.body if is_xai_component(node)] - - def extract_component(self, node: ast.ClassDef, file_path, file_lines, python_path): - name = node.name - - keywords = {kw.arg: kw.value.value for kw in chain.from_iterable(decorator.keywords - for decorator in node.decorator_list - if isinstance(decorator, ast.Call) and decorator.func.id == "xai_component")} - - # Group Name for Display - category = remove_prefix(file_path.parent.name, "xai_").upper() - - is_arg = lambda n: isinstance(n, ast.AnnAssign) and \ - isinstance(n.annotation, ast.Subscript) and \ - n.annotation.value.id in ['InArg', 'InCompArg', 'OutArg'] - - is_flow_arg = lambda n: isinstance(n, ast.AnnAssign) and \ - isinstance(n.annotation, ast.Name) and \ - n.annotation.id in ['BaseComponent'] - - python_version = platform.python_version_tuple() - - variables = [] - for v in (node.body): - if is_flow_arg(v): - variables.append({ - "name": v.target.id, - "kind": v.annotation.id, - }) - continue - elif is_arg(v): - variables.append({ - "name": v.target.id, - "kind": v.annotation.value.id, - "type": read_orig_code(v.annotation.slice.value if int(python_version[1]) == 8 else v.annotation.slice, file_lines) - }) - continue - - docstring = ast.get_docstring(node) - lineno = [ - { - "lineno": node.lineno, - "end_lineno": node.end_lineno - } - ] - - output = { - "class": name, - "package_name": ("xai_components." if python_path is None else "") + file_path.as_posix().replace("/", ".")[:-3], - "python_path": str(python_path) if python_path is not None else None, - "abs_file_path": os.path.join(str(python_path), str(file_path)) if python_path is not None else None, - "file_path": "xai_components/" + (file_path.as_posix()[:-3] + ".py" if platform.system() == "Windows" else str(file_path)), - "task": name, - "header": GROUP_ADVANCED, - "category": category, - "type": "debug", - "variables": variables, - "docstring": docstring, - "lineno" : lineno - } - output.update(keywords) - - return output + self.finish(self.component_parser.get())