diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..1ce82f3 --- /dev/null +++ b/TODO.md @@ -0,0 +1,8 @@ +## People + +- [ ] use remaining fields +- [ ] Once a day, if there are new commits in openstates/people, re-create. + +## Other + +- [ ] expand data types diff --git a/src/ossql/people_to_sqlite.py b/src/ossql/people_to_sqlite.py index 3b07e7e..e8ce00c 100644 --- a/src/ossql/people_to_sqlite.py +++ b/src/ossql/people_to_sqlite.py @@ -5,8 +5,6 @@ from .schemas.common import db from .schemas.people import Person, PersonLink, PersonSource, PersonRole, PersonOffice - - def to_links(person, dict_list, cls): for dl in dict_list: cls.create( @@ -15,7 +13,8 @@ def to_links(person, dict_list, cls): note=dl.pop("note", ""), ) -def load_people_yaml(dir_path: pathlib.Path): + +def load_people_yaml(dir_path: pathlib.Path) -> int: # ensure all tables exist db.create_tables([Person, PersonLink, PersonSource, PersonRole, PersonOffice]) @@ -49,9 +48,9 @@ def load_people_yaml(dir_path: pathlib.Path): person=person, jurisdiction=role.pop("jurisdiction"), district=role.pop("district", ""), - type=role.pop('type'), - start_date=role.pop('start_date', None), - end_date=role.pop('end_date', None), + type=role.pop("type"), + start_date=role.pop("start_date", None), + end_date=role.pop("end_date", None), ) for office in pdata.pop("offices", []): PersonOffice.create( @@ -59,7 +58,7 @@ def load_people_yaml(dir_path: pathlib.Path): classification=office.pop("classification"), address=office.pop("address", ""), voice=office.pop("voice", ""), - fax=office.pop("fax",""), + fax=office.pop("fax", ""), ) # currently not using other_names, other_identifiers @@ -72,7 +71,6 @@ def load_people_yaml(dir_path: pathlib.Path): return created - if __name__ == "__main__": path = pathlib.Path(sys.argv[1]) n_people = 0 diff --git a/src/ossql/schemas/common.py b/src/ossql/schemas/common.py index ec651ef..b083b52 100644 --- a/src/ossql/schemas/common.py +++ b/src/ossql/schemas/common.py @@ -1,14 +1,16 @@ -from peewee import SqliteDatabase, Model +from peewee import Model from playhouse.sqlite_ext import SqliteExtDatabase -db = SqliteExtDatabase('openstates.db', pragmas=( - ('cache_size', 1024 * 64), # 64MB page-cache. - ('journal_mode', 'wal'), # Use WAL-mode (you should always use this!). - ('foreign_keys', 1)) -) +db = SqliteExtDatabase( + "openstates.db", + pragmas=( + ("cache_size", 1024 * 64), # 64MB page-cache. + ("journal_mode", "wal"), # Use WAL-mode (you should always use this!). + ("foreign_keys", 1), + ), +) class BaseModel(Model): class Meta: database = db - diff --git a/src/ossql/schemas/people.py b/src/ossql/schemas/people.py index b55081e..28dca59 100644 --- a/src/ossql/schemas/people.py +++ b/src/ossql/schemas/people.py @@ -2,6 +2,7 @@ from .common import BaseModel from peewee import ForeignKeyField, TextField, DateField from playhouse.sqlite_ext import JSONField + class Person(BaseModel): id = TextField(primary_key=True) name = TextField() @@ -13,6 +14,7 @@ class Person(BaseModel): party = TextField() extras = JSONField() + class PersonLink(BaseModel): person = ForeignKeyField(Person, backref="links") url = TextField() @@ -24,6 +26,7 @@ class PersonSource(BaseModel): url = TextField() note = TextField() + class PersonRole(BaseModel): person = ForeignKeyField(Person, backref="roles") start_date = DateField(null=True) @@ -32,6 +35,7 @@ class PersonRole(BaseModel): jurisdiction = TextField() district = TextField() + class PersonOffice(BaseModel): person = ForeignKeyField(Person, backref="offices") classification = TextField()