change recipe to have a method to obtain graph data and use it in show
This commit is contained in:
parent
20f9809689
commit
046c9e453b
@ -10,7 +10,6 @@ from scrapeghost.preprocessors import CleanHTML
|
|||||||
|
|
||||||
|
|
||||||
async def add_response(obj_with_url):
|
async def add_response(obj_with_url):
|
||||||
print(obj_with_url["url"])
|
|
||||||
url = obj_with_url["url"]
|
url = obj_with_url["url"]
|
||||||
async with httpx.AsyncClient() as client:
|
async with httpx.AsyncClient() as client:
|
||||||
response = await client.get(url)
|
response = await client.get(url)
|
||||||
|
@ -2,6 +2,7 @@ import importlib
|
|||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
import typer
|
import typer
|
||||||
import sys
|
import sys
|
||||||
|
from pprint import pprint
|
||||||
from typing import List, Optional
|
from typing import List, Optional
|
||||||
from typing_extensions import Annotated
|
from typing_extensions import Annotated
|
||||||
|
|
||||||
@ -47,6 +48,11 @@ def show(ctx: typer.Context):
|
|||||||
ctx.obj.show()
|
ctx.obj.show()
|
||||||
|
|
||||||
|
|
||||||
|
@app.command()
|
||||||
|
def graph(ctx: typer.Context):
|
||||||
|
pprint(ctx.obj.graph_data())
|
||||||
|
|
||||||
|
|
||||||
@app.command()
|
@app.command()
|
||||||
def run(
|
def run(
|
||||||
ctx: typer.Context,
|
ctx: typer.Context,
|
||||||
|
@ -166,20 +166,17 @@ class Recipe:
|
|||||||
return beaker
|
return beaker
|
||||||
|
|
||||||
def show(self):
|
def show(self):
|
||||||
for node in networkx.topological_sort(self.graph):
|
graph_data = self.graph_data()
|
||||||
beaker = self.beakers[node]
|
for node in graph_data:
|
||||||
temp = isinstance(beaker, TempBeaker)
|
if node["temp"]:
|
||||||
if temp:
|
typer.secho(node["name"], fg=typer.colors.CYAN)
|
||||||
typer.secho(node, fg=typer.colors.CYAN)
|
|
||||||
else:
|
else:
|
||||||
lb = len(beaker)
|
|
||||||
typer.secho(
|
typer.secho(
|
||||||
f"{node} ({lb})",
|
f"{node['name']} ({node['len']})",
|
||||||
fg=typer.colors.GREEN if lb else typer.colors.YELLOW,
|
fg=typer.colors.GREEN if node["len"] else typer.colors.YELLOW,
|
||||||
)
|
)
|
||||||
for from_b, to_b, edge in self.graph.out_edges(node, data=True):
|
for edge in node["edges"]:
|
||||||
name = edge["transform"].name
|
print(f" -({edge['transform'].name})-> {edge['to_beaker']}")
|
||||||
print(f" {from_b} -({name})-> {to_b}")
|
|
||||||
for k, v in edge["transform"].error_map.items():
|
for k, v in edge["transform"].error_map.items():
|
||||||
if isinstance(k, tuple):
|
if isinstance(k, tuple):
|
||||||
typer.secho(
|
typer.secho(
|
||||||
@ -189,6 +186,33 @@ class Recipe:
|
|||||||
else:
|
else:
|
||||||
typer.secho(f" {k.__name__} -> {v}", fg=typer.colors.RED)
|
typer.secho(f" {k.__name__} -> {v}", fg=typer.colors.RED)
|
||||||
|
|
||||||
|
def graph_data(self):
|
||||||
|
nodes = {}
|
||||||
|
|
||||||
|
for node in networkx.topological_sort(self.graph):
|
||||||
|
beaker = self.beakers[node]
|
||||||
|
temp = isinstance(beaker, TempBeaker)
|
||||||
|
|
||||||
|
nodes[node] = {
|
||||||
|
"name": node,
|
||||||
|
"temp": temp,
|
||||||
|
"len": len(beaker),
|
||||||
|
"edges": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
rank = 0
|
||||||
|
for from_b, to_b, edge in self.graph.in_edges(node, data=True):
|
||||||
|
if nodes[from_b]["rank"] > rank:
|
||||||
|
rank = nodes[from_b]["rank"]
|
||||||
|
nodes[node]["rank"] = rank + 1
|
||||||
|
|
||||||
|
for from_b, to_b, edge in self.graph.out_edges(node, data=True):
|
||||||
|
edge["to_beaker"] = to_b
|
||||||
|
nodes[node]["edges"].append(edge)
|
||||||
|
|
||||||
|
# all data collected for display
|
||||||
|
return sorted(nodes.values(), key=lambda x: (x["rank"], x["name"]))
|
||||||
|
|
||||||
def run_once(
|
def run_once(
|
||||||
self, start_beaker: str | None = None, end_beaker: str | None = None
|
self, start_beaker: str | None = None, end_beaker: str | None = None
|
||||||
) -> None:
|
) -> None:
|
||||||
|
Loading…
Reference in New Issue
Block a user