36 lines
928 B
Python
36 lines
928 B
Python
import httpx
|
|
from beakers.beakers import Beaker
|
|
from beakers.recipe import Recipe
|
|
|
|
|
|
async def add_response(obj_with_url):
|
|
print(obj_with_url["url"])
|
|
url = obj_with_url["url"]
|
|
response = await httpx.get(url)
|
|
return {
|
|
"url": url,
|
|
"status_code": response.status_code,
|
|
"response_body": response.text,
|
|
}
|
|
|
|
|
|
# current thinking, beakers exist within a recipe
|
|
recipe = Recipe("fetch urls", "url_example.db")
|
|
recipe.add_beaker("agencies")
|
|
recipe.add_beaker("responses")
|
|
recipe.add_beaker("bad_requests")
|
|
recipe.add_beaker("good_urls", temp=True)
|
|
recipe.add_beaker("missing_urls", temp=True)
|
|
recipe.add_conditional(
|
|
"agencies",
|
|
lambda x: x["url"].startswith("http"),
|
|
if_true="good_urls",
|
|
if_false="missing_urls",
|
|
)
|
|
recipe.add_transform(
|
|
"good_urls", "responses", add_response, error_beaker="bad_requests"
|
|
)
|
|
|
|
recipe.csv_to_beaker("agencies.csv", "agencies")
|
|
recipe.run_once()
|