2023-04-27 06:25:07 +00:00
|
|
|
import csv
|
|
|
|
from beakers.recipe import Recipe
|
|
|
|
from beakers.beaker import Beaker
|
|
|
|
|
|
|
|
urls = Beaker("urls")
|
|
|
|
responses = Beaker("responses")
|
|
|
|
|
|
|
|
|
|
|
|
async def add_response(obj_with_url):
|
|
|
|
url = obj_with_url["url"]
|
|
|
|
response = await httpx.get(url)
|
|
|
|
return {
|
|
|
|
"url": url,
|
|
|
|
"status_code": response.status_code,
|
|
|
|
"response_body": response.text,
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2023-04-27 19:51:25 +00:00
|
|
|
agencies = Beaker("agencies")
|
2023-04-27 06:25:07 +00:00
|
|
|
responses = Beaker("responses")
|
|
|
|
recipe = Recipe("fetch urls")
|
2023-04-27 19:51:25 +00:00
|
|
|
recipe.csv_to_beaker("agencies.csv", agencies)
|
2023-04-27 06:25:07 +00:00
|
|
|
recipe.add_pour(agencies, responses, add_response)
|
|
|
|
|
|
|
|
recipe.run_linearly()
|