{ "meta": { "doc": "data-reality", "generator": "civic_data/datadict/generator.py", "config": "civic_data/datadict/config.py", "lineage_config": "civic_data/datadict/lineage_config.py", "project": { "name": "civic-data", "city": "Louisville / Jefferson County, KY", "city_key": "louisville", "geography_unit": "ZIP code (41-ZIP Louisville boundary set)", "summary": "Cleaned, normalized, and documented Louisville public open data. This document is the as-built data reality — what we actually hold, what it honestly measures, how much to trust it, and (in the lineage half) exactly what we did to it and why. It is regenerated from the live database + ingest code; the numbers cannot drift from it.", "maturity_posture": "v1 — generated dictionary + transformation lineage over crime, 311, the Economic-Distress theme (land bank sales, code-enforcement liens, foreclosure filings), the Investment & Growth theme (food service, ABC licenses, restaurant inspections, construction permits), Housing & Affordability (short-term rental registrations), and the geography/reference spine (district/neighborhood/HUD boundaries, Census ACS population, ZIP↔geo crosswalks)." }, "as_of": "2026-07-12", "data_version": "2026-07-12+27720116", "generated_at": "2026-07-13T01:46:56.006224+00:00", "provenance": [ { "source_dataset": "311-service-requests-2010", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Louisville_Service_Requests_2010/FeatureServer/0", "source_rowcount": 106380, "loaded": 106380, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "311-service-requests-2011", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Louisville_311_Service_Requests_2011/FeatureServer/0", "source_rowcount": 106296, "loaded": 106296, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "311-service-requests-2012", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/311_Service_Requests/FeatureServer/0", "source_rowcount": 105503, "loaded": 105503, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "311-service-requests-2013", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Louisville_311_Service_Requests_2013/FeatureServer/0", "source_rowcount": 96335, "loaded": 96335, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "311-service-requests-2014", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Louisville_311_Service_Requests_2014/FeatureServer/0", "source_rowcount": 94605, "loaded": 94605, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "311-service-requests-2015", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Louisville_311_Service_Requests_2015/FeatureServer/0", "source_rowcount": 104145, "loaded": 104145, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "311-service-requests-2016", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Louisville_311_Service_Requests_2016/FeatureServer/0", "source_rowcount": 102039, "loaded": 102039, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "311-service-requests-2017", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Louisville_311_Service_Requests_2017/FeatureServer/0", "source_rowcount": 102135, "loaded": 102135, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "311-service-requests-2018", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Louisville_311_Service_Requests_2018/FeatureServer/0", "source_rowcount": 124741, "loaded": 124741, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "311-service-requests-2019a", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Louisville_311_Service_Requests_Jan_1st_2019_until_Jun_14_2019/FeatureServer/0", "source_rowcount": 56917, "loaded": 56917, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "311-service-requests-2019b", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/311_Service_Requests_June_18th_2019_to_December_18th_2019/FeatureServer/0", "source_rowcount": 95786, "loaded": 95786, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "311-service-requests-2020", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Louisville_311_Service_Requests_2020/FeatureServer/0", "source_rowcount": 229875, "loaded": 229875, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "311-service-requests-2021", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Louisville_Metro_KY_311_Service_Requests_2021/FeatureServer/0", "source_rowcount": 216399, "loaded": 216399, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "311-service-requests-2022", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/metro_311_2022/FeatureServer/0", "source_rowcount": 175152, "loaded": 175152, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "311-service-requests-2023", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/metro_call_311_service_requests_2023/FeatureServer/0", "source_rowcount": 173246, "loaded": 173246, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "311-service-requests-2024", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Louisville_Metro_KY_Metro_311_Service_Requets_2024/FeatureServer/0", "source_rowcount": 169601, "loaded": 169601, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "311-service-requests-2025", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/metro_311_2025/FeatureServer/0", "source_rowcount": 182490, "loaded": 182490, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "311-service-requests-2026", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/metro_311_2026/FeatureServer/0", "source_rowcount": 102179, "loaded": 102179, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "abc-local-licenses", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/abc_license_ait4135/FeatureServer/0", "source_rowcount": 1681, "loaded": 1681, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "abc-state-licenses", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/ABC_State_ActiveLicenses/FeatureServer/0", "source_rowcount": 5483, "loaded": 5483, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "active-construction-permits", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/active_construction_permits/FeatureServer/0", "source_rowcount": 23332, "loaded": 23332, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "crime-data-2007", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Crime_Data_20071/FeatureServer/0", "source_rowcount": 90767, "loaded": 90767, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "crime-data-2008", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Crime_Data_20081/FeatureServer/0", "source_rowcount": 86481, "loaded": 86481, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "crime-data-2009", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Crime_Data_2009/FeatureServer/0", "source_rowcount": 78380, "loaded": 78380, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "crime-data-2010", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Crime_Data_2010/FeatureServer/0", "source_rowcount": 82871, "loaded": 82871, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "crime-data-2011", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Crime_Data_2011/FeatureServer/0", "source_rowcount": 86013, "loaded": 86013, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "crime-data-2012", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Crime_Data_2012/FeatureServer/0", "source_rowcount": 81406, "loaded": 81406, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "crime-data-2013", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Crime_Data_2013/FeatureServer/0", "source_rowcount": 77127, "loaded": 77127, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "crime-data-2014", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Crime_Data_2014/FeatureServer/0", "source_rowcount": 77466, "loaded": 77466, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "crime-data-2015", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Crime_Data_2015/FeatureServer/0", "source_rowcount": 79839, "loaded": 79839, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "crime-data-2016", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Crime_Data_2016/FeatureServer/0", "source_rowcount": 84535, "loaded": 84535, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "crime-data-2017", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Crime_Data_2017/FeatureServer/0", "source_rowcount": 82113, "loaded": 82113, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "crime-data-2018", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Crime_Data_2018_/FeatureServer/0", "source_rowcount": 80000, "loaded": 80000, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "crime-data-2019", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/CRIME_DATA2019/FeatureServer/0", "source_rowcount": 73420, "loaded": 73420, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "crime-data-2020", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/crime_2020/FeatureServer/0", "source_rowcount": 70233, "loaded": 70233, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "crime-data-2021", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Louisville_Metro_KY_Crime_Data_2021/FeatureServer/0", "source_rowcount": 72127, "loaded": 72127, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "crime-data-2022", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Louisville_Metro_KY_Crime_Data_2022/FeatureServer/0", "source_rowcount": 71506, "loaded": 71506, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "crime-data-2023", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/crime_data_2023/FeatureServer/0", "source_rowcount": 71502, "loaded": 71502, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "crime-data-2024", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/crimedata2024/FeatureServer/0", "source_rowcount": 70914, "loaded": 70914, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "crime-data-2025", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/crime_data_2025/FeatureServer/0", "source_rowcount": 70279, "loaded": 70279, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "crime-data-2026", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/crime_data_2026/FeatureServer/0", "source_rowcount": 32670, "loaded": 32670, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "food-service-establishments", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Louisville_Metro_KY_Permitted_Food_Service_Establishments/FeatureServer/0", "source_rowcount": 3366, "loaded": 3366, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "landbank-historical", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Landbank_Sales_Historical_Data/FeatureServer/0", "source_rowcount": 418, "loaded": 418, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "lien-final-orders", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/lien_notification_final_orders/FeatureServer/0", "source_rowcount": 434, "loaded": 434, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "property-foreclosures", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/Louisville_Metro_KY_Property_Foreclosures/FeatureServer/0", "source_rowcount": 2876, "loaded": 3448, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "restaurant-inspection-scores", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/FoodServiceData/FeatureServer/0", "source_rowcount": 11273, "loaded": 11273, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "str-registration", "source_url": "https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/short_term_rental_registration/FeatureServer/0", "source_rowcount": 1252, "loaded": 1252, "status": "ok", "pulled_at": "2026-07-09" }, { "source_dataset": "zip_population", "source_url": "https://api.census.gov/data/{year}/acs/acs5 (vintages 2023..2023)", "source_rowcount": 40, "loaded": 40, "status": "ok", "pulled_at": "2026-07-11" } ], "feature_flags": { "has_coverage_model": false, "has_postgis": true, "has_quarantine": true, "provenance_style": "api", "year_semantics": "calendar" }, "trust_tiers": { "trendable": "Clean, full multi-year coverage — safe for time series and trend claims.", "viewable_caveated": "Real data with gaps/cliffs/discontinuities — viewable within a stated window, must not be presented as an uncaveated trend.", "snapshot_only": "No honest time series — current-state snapshot only.", "unsupported": "Claims the data cannot back — named so nobody proposes vaporware on top." }, "pii_tiers": { "public-aggregate-safe": "No individuals named; safe to surface at aggregate AND record level.", "sensitive-aggregate": "Public record, but granular republication at scale is a different posture than the source's — surface aggregate only.", "person-naming": "Names identifiable individuals. Not present in this DB (no-PII policy strips person-name fields at ingest); listed for completeness." }, "with_source_audit": false }, "loss_boundary": { "lead": "What we DROPPED between source and silver. If bronze raw is not persisted, the silver.quarantine table is the ONLY visible drop — losses before the ZIP-validation step (fetch gaps, parse failures, dedup) are invisible from the DB. The source-count audit (datadict/audit.py, folded in below) bounds the dedup-vs-loss question from the source side.", "quarantine": { "note": "silver.quarantine is the visible drop. Losses before the ZIP-validation step are not captured here unless bronze raw is persisted.", "total": 20823, "by_source": [ { "source_table": "silver.crime_incidents", "count": 20484 }, { "source_table": "silver.service_requests", "count": 178 }, { "source_table": "silver.lien_orders", "count": 136 }, { "source_table": "silver.abc_licenses", "count": 15 }, { "source_table": "silver.construction_permits", "count": 8 }, { "source_table": "silver.property_foreclosures", "count": 2 } ], "by_reason": [ { "reason": "ZIP not in Louisville boundary set", "count": 20823 } ] }, "headline_findings": [ { "title": "Crime citywide undercount — location-less records quarantined", "body": "The single largest real drop. LMPD stamps location-less offenses (fraud, identity theft, cyber — crimes with no physical address) with placeholder ZIP 40056 (Pewee Valley, outside the 41-ZIP set), so validation quarantines them. Per-ZIP crime is unaffected (they can't be placed anyway), but CITYWIDE crime totals undercount and the gap is biased toward non-geographic offense types.", "live": { "quarantined_at_40056": 19734, "crime_silver": 1242777, "crime_quarantined_total": 20484, "citywide_undercount_pct": 1.6 } }, { "title": "311-vs-crime NULL-zip inconsistency", "body": "311 KEEPS its missing-ZIP records (NULL zip, recoverable via lat/lng); crime QUARANTINES its location-less records entirely. Same situation, opposite treatment. A candidate normalization is to distinguish 'real Louisville record, no location' (→ NULL zip, keep) from 'another jurisdiction' (→ quarantine). Documented, not yet changed." }, { "title": "Bronze persistence determines auditability", "body": "If raw bronze is not retained, the quarantine step is the only place a drop becomes visible; earlier-stage losses (fetch/parse/dedup) would not show up at all. The from-source reproduction recipe + the source-count audit together bound this.", "live": { "tables": { "business_raw": 10530, "crime_raw": 1524649, "inspections_raw": 11273, "property_raw": 28884, "service_requests_raw": 2348824 }, "all_empty": false } } ], "audit_verdict": "Frozen per-year crime/311 services barely grow, so a source-vs-silver gap on those years is a clean loss check. Large apparent 'gaps' on crime are legitimate offense-row deduplication (incident counts sound), not lost data; the one genuine drop is the location-less crime at 40056.", "source_audit": null }, "concepts": [ { "key": "safety", "title": "Safety & Enforcement", "trust_tier": "trendable", "backed_by": [ { "domain": "crime", "fields": [ "offense_type", "incident_date", "zip_code" ], "role": "primary — reported-incident density + violent/property split" } ], "future_sources": [ "traffic fatalities", "uniform citations", "fire incidents" ], "honest_claim": "Can speak to REPORTED crime per ZIP with the policing-bias caveat. Cannot speak to 'how safe' a place is, nor to under-reported offense types." }, { "key": "livability", "title": "Civic Responsiveness & Livability", "trust_tier": "viewable_caveated", "backed_by": [ { "domain": "service_requests", "fields": [ "service_name", "requested_datetime", "status" ], "role": "primary — 311 demand + resolution" } ], "future_sources": [ "noise citations", "property-maintenance cases" ], "honest_claim": "Can speak to 311 demand within 2010–2018 (the trendable window), with the reporting-propensity bias caveat. resolution_days unusable pre-2022." }, { "key": "economic_distress", "title": "Economic Distress & Property Disinvestment", "trust_tier": "viewable_caveated", "backed_by": [ { "domain": "property_foreclosures", "fields": [ "action_filed", "zip_code", "neighborhood" ], "role": "primary — foreclosure FILING density (dense from ~2013, no Great-Recession era)" }, { "domain": "lien_orders", "fields": [ "notification_date", "zip_code", "final_order_state" ], "role": "supporting — code-enforcement lien SNAPSHOT (rolling window, not a trend)" }, { "domain": "landbank_sales", "fields": [ "sale_date", "zip_code", "sale_program" ], "role": "supporting — public disposition of blighted parcels (historical, small N)" } ], "future_sources": [ "property tax delinquency", "vacant/abandoned property registry", "eviction filings" ], "honest_claim": "Can speak to foreclosure-FILING density per ZIP from ~2013 on (not completed loss, no pre-2013 wave), cross-referenced with a current lien snapshot and historical land-bank dispositions. Cannot present liens as a time series (rolling-window source) nor claim the pre-2013 foreclosure history." }, { "key": "investment_growth", "title": "Investment & Growth", "trust_tier": "viewable_caveated", "backed_by": [ { "domain": "construction_permits", "fields": [ "issue_date", "project_costs", "zip_code" ], "role": "primary — active-permit density + declared project cost (active set, 2020–22 caveat)" }, { "domain": "abc_licenses", "fields": [ "license_type", "status", "zip_code" ], "role": "supporting — current alcohol-license footprint (nightlife/dining)" }, { "domain": "food_service_establishments", "fields": [ "facility_type_description", "zip_code" ], "role": "supporting — current permitted food-service roster" }, { "domain": "restaurant_inspections", "fields": [ "score", "grade", "inspection_date" ], "role": "supporting — recent inspection scores (rolling ~12-month window)" } ], "future_sources": [ "right-of-way permits", "business licenses", "certificates of occupancy" ], "honest_claim": "Can speak to CURRENT investment/commercial activity per ZIP — active construction permits, alcohol licenses, permitted food service, recent inspection scores. All four are current-state or rolling-window sources: viewable now, but not a clean multi-year trend from a single pull, and permit years 2020–2022 carry a source capture discontinuity." }, { "key": "housing_affordability", "title": "Housing & Affordability", "trust_tier": "snapshot_only", "backed_by": [ { "domain": "str_licenses", "fields": [ "date_opened", "zip_code", "council_district" ], "role": "primary — registered short-term-rental density (supply-side housing pressure)" } ], "future_sources": [ "evictions", "rental registrations", "housing code violations", "HUD Fair Market Rent" ], "honest_claim": "Can speak to REGISTERED short-term-rental density per ZIP as a supply-side housing- pressure signal — a current snapshot, understating the unlicensed STR footprint. Not yet a full housing- affordability picture (rents, evictions, and code violations are future sources)." } ], "mdm": [ { "domain_key": "zip_codes", "kind": "master-reference", "title": "Canonical Louisville geography (ZIP boundary set)", "source_system": "Louisville / LOJIC ArcGIS Hub — 'Jefferson County KY ZIP Codes' Feature Service.", "silver_table": "silver.zip_codes", "why_summary": "One governed canonical geography that every transactional dataset validates against. This is master data, not a transaction stream: it is the authority that decides whether a crime or 311 record has a valid Louisville ZIP.", "governance": [ "Integrity check: EXPECTED_ZIP_COUNT = 41 distinct ZIPs; ingest refuses to serve a whitelist below this count (adapters.validation.load_valid_zips).", "Authority: adapters.validation.validate_zip resolves every event record's ZIP against this set (valid → keep, missing → NULL, out-of-area → quarantine).", "Versioning: boundaries are a versioned reference — re-pulling REPLACES the set (DELETE + reload). Boundary changes are a governed event, not an append." ], "transforms": [ { "op": "fetch", "detail": "ArcGIS GeoJSON query (42 ZIP polygons with geometry).", "why": "Authoritative boundaries from the city's own portal, not derived.", "loss": "None." }, { "op": "normalize_fields", "detail": "ZIPCODE padded to 5 digits.", "why": "Consistent 5-digit keys for joins/validation.", "loss": "None." }, { "op": "dedup", "detail": "ON CONFLICT(zip) load → 41 distinct ZIPs (source reports 42; one is a duplicate geometry).", "why": "One row per canonical ZIP.", "loss": "The duplicate geometry collapses to one row (intended)." } ], "produces_columns": [ "zip", "geom" ], "reproduce": { "commands": [ "civic-data ingest-zips" ], "gaps": [] }, "verified_against_schema": true, "note": null }, { "domain_key": "council_districts", "kind": "master-reference", "title": "Metro Council district boundaries (versioned)", "source_system": "LOJIC MapServer — OpenDataPolitical/8 (Council Districts, GeoJSON).", "silver_table": "silver.council_districts", "why_summary": "The district dimension every event table rolls up to. Master geography, version-scoped: a redistricting is a governed event, not an append — the current boundaries are the 2022-redistricting version. council_member is an ELECTED official (public record), kept by policy.", "governance": [ "Integrity invariant: exactly 26 districts per version, or the load rolls back.", "At most one ACTIVE version (effective_to IS NULL) per district_number; setting effective_to on a prior version is a separate, deliberate redistricting step.", "Version-scoped: re-ingest REPLACES only the named version's rows (+ its crosswalk); prior versions are preserved." ], "transforms": [ { "op": "fetch", "detail": "GeoJSON query (COUNDIST, COUN_NAME) from the LOJIC MapServer.", "why": "Authoritative boundaries from the city's GIS, not derived.", "loss": "None." }, { "op": "build_geom", "detail": "ST_Multi(ST_MakeValid(ST_GeomFromGeoJSON)) → MultiPolygon 4326.", "why": "One valid MultiPolygon type for spatial joins.", "loss": "None — geometry repair only." }, { "op": "normalize_fields", "detail": "COUNDIST → district_number (int), COUN_NAME → council_member; stamped version_label=2022-redistricting, effective_from=2022-01-01, effective_to=NULL.", "why": "Typed, versioned district rows.", "loss": "None." } ], "produces_columns": [ "district_number", "council_member", "geom", "version_label", "effective_from", "effective_to" ], "reproduce": { "commands": [ "civic-data ingest-zips # canonical ZIP set first", "civic-data ingest-geo # boundaries + crosswalks (also builds zip_to_district)" ], "gaps": [ "Reproduces civic-graph exactly (26 = 26)." ] }, "verified_against_schema": true, "note": null }, { "domain_key": "neighborhoods", "kind": "master-reference", "title": "Neighborhood boundaries", "source_system": "LOJIC ArcGIS FeatureServer — LOUISVILLE_NEIGHBORHOODS/44 (GeoJSON).", "silver_table": "silver.neighborhoods", "why_summary": "The neighborhood dimension for sub-ZIP rollups. Master geography; a full-refresh reference set (DELETE + reload), not an append stream.", "governance": [ "Full-refresh: re-ingest REPLACES the set (DELETE + reload, id sequence reset) — a boundary change is a governed replacement.", "UNIQUE(name): one row per named neighborhood (source's duplicate-name features collapse)." ], "transforms": [ { "op": "fetch", "detail": "GeoJSON query (NH_NAME, NH_TYPE) from the LOJIC FeatureServer.", "why": "Authoritative boundaries from the city's GIS.", "loss": "Features with an empty name are skipped." }, { "op": "build_geom", "detail": "ST_Multi(ST_MakeValid(ST_GeomFromGeoJSON)) → MultiPolygon 4326.", "why": "One valid MultiPolygon type.", "loss": "None — geometry repair only." }, { "op": "dedup", "detail": "ON CONFLICT(name) upsert; 151 source features → 150 rows.", "why": "One row per neighborhood name.", "loss": "A duplicate-name feature collapses to one (intended)." } ], "produces_columns": [ "id", "name", "nh_type", "geom" ], "reproduce": { "commands": [ "civic-data ingest-geo # boundaries + crosswalks (also builds zip_to_neighborhood)" ], "gaps": [ "Reproduces civic-graph exactly (150 = 150)." ] }, "verified_against_schema": true, "note": null }, { "domain_key": "hud_zones", "kind": "master-reference", "title": "HUD 1979 grant-zone boundaries", "source_system": "LOJIC MapServer — OpenDataSociety/15 (HUD 1979 Grant Zones, GeoJSON).", "silver_table": "silver.hud_zones", "why_summary": "A legacy grant-zone dimension (urban-core coverage). Master geography; multi-part zones are unioned into one MultiPolygon per code.", "governance": [ "Full-refresh: re-ingest REPLACES the set (DELETE + reload).", "One row per nh_code: disjoint source polygons sharing a code are ST_Union'd together (e.g. code 99 'REMAINDER OF CITY' has many parts)." ], "transforms": [ { "op": "fetch", "detail": "GeoJSON query (NH_CODE, NH_NAME) from the LOJIC MapServer (92 parts).", "why": "Authoritative legacy grant zones from the city's GIS.", "loss": "None." }, { "op": "build_geom", "detail": "ST_Multi(ST_MakeValid(ST_GeomFromGeoJSON)); parts sharing a code are ST_Union'd on conflict.", "why": "One valid MultiPolygon per zone code.", "loss": "None — the 92 source parts union to 71 zones (intended)." } ], "produces_columns": [ "nh_code", "name", "geom" ], "reproduce": { "commands": [ "civic-data ingest-geo # boundaries + crosswalks (also builds zip_to_hud_zone)" ], "gaps": [ "Reproduces civic-graph exactly (71 zones from 92 parts). HUD zones cover the urban core only (~20 of 41 ZIPs), by design." ] }, "verified_against_schema": true, "note": null }, { "domain_key": "zip_population", "kind": "master-reference", "title": "Census ACS population by ZIP (per-capita denominator)", "source_system": "U.S. Census Bureau — ACS 5-Year Estimates at ZCTA level (api.census.gov), vintages 2011–2023.", "silver_table": "silver.zip_population", "why_summary": "The per-capita denominator for every rate. Population + median age + median household income per (ZIP, ACS vintage), so a count can be normalized with a year-appropriate population instead of one static number.", "governance": [ "Scoped to the canonical 41-ZIP set (silver.zip_codes) — out-of-area ZCTAs are never queried.", "ZIP ≈ ZCTA (not exact) — documented, accepted for per-capita use.", "Pre-2020 vintages use 2010 ZCTAs (require in=state:21); 2020+ use 2020 ZCTAs.", "Suppressed/negative Census sentinels → NULL (never a guessed value)." ], "transforms": [ { "op": "fetch", "detail": "ACS 5-year API per vintage (B01001_001E population, B01002_001E median age, B19013_001E median household income), batched by 50 ZCTAs.", "why": "Year-appropriate population for historical per-capita rates.", "loss": "40225 has no ZCTA population (non-ZCTA ZIP) → absent; 40 of 41 ZIPs return per vintage." }, { "op": "normalize_fields", "detail": "Suppressed/None/negative → NULL; typed to int/numeric.", "why": "Zero-vs-NULL discipline: NULL means no usable estimate, never 0.", "loss": "Suppressed cells become NULL (honest unknown)." }, { "op": "dedup", "detail": "ON CONFLICT(zip_code, vintage) upsert.", "why": "One row per ZIP per ACS vintage.", "loss": "None." } ], "produces_columns": [ "zip_code", "vintage", "population", "median_age", "median_household_income", "updated_at" ], "reproduce": { "commands": [ "civic-data ingest-zips # canonical ZIP set first", "civic-data ingest-population # ACS 2011–2023 (CENSUS_API_KEY optional at this volume)" ], "gaps": [ "520 rows (40 ZCTAs × 13 vintages) vs civic-graph's 600 — CLEANER, not different: the 520 shared (ZIP, vintage) cells match civic-graph's values exactly; civic-graph's extra 80 rows are ~20 non-Louisville ZCTAs (LA/Chicago/Indianapolis/Frankfort…) pulled from a broader gold ZIP list. Scoping to the canonical 41-ZIP set correctly excludes them. 2009–2010 unavailable at ZCTA level; 40225 has no Census population." ] }, "verified_against_schema": true, "note": null } ], "domains": [ { "key": "crime", "title": "Crime incidents (LMPD / NIBRS)", "trust_tier": "trendable", "generated": { "schema": "silver", "table": "crime_incidents", "row_count": 1242777, "primary_key": [ "id" ], "year_column": "incident_date", "fields": [ { "name": "id", "type": "bigint", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "source_id", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "incident_date", "type": "date", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "incident_time", "type": "time without time zone", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "offense_type", "type": "text", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "offense_description", "type": "text", "nullable": true, "null_count": 2, "null_pct": 0.0 }, { "name": "location_address", "type": "text", "nullable": true, "null_count": 5120, "null_pct": 0.4 }, { "name": "latitude", "type": "double precision", "nullable": true, "null_count": 40378, "null_pct": 3.2 }, { "name": "longitude", "type": "double precision", "nullable": true, "null_count": 40378, "null_pct": 3.2 }, { "name": "geom", "type": "USER-DEFINED", "nullable": true }, { "name": "census_tract_geoid", "type": "text", "nullable": true, "null_count": 1242777, "null_pct": 100.0 }, { "name": "neighborhood_id", "type": "integer", "nullable": true, "null_count": 1242777, "null_pct": 100.0 }, { "name": "zip_code", "type": "text", "nullable": true, "null_count": 3522, "null_pct": 0.3 }, { "name": "premise_type", "type": "text", "nullable": true, "null_count": 5001, "null_pct": 0.4 }, { "name": "weapon_used", "type": "text", "nullable": true, "null_count": 1242777, "null_pct": 100.0 }, { "name": "status", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "source_dataset", "type": "text", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "bronze_id", "type": "bigint", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "zip_derived", "type": "boolean", "nullable": false, "null_count": 0, "null_pct": 0.0 } ], "per_year": [ { "year": 1970, "count": 4 }, { "year": 1971, "count": 5 }, { "year": 1972, "count": 5 }, { "year": 1973, "count": 3 }, { "year": 1974, "count": 4 }, { "year": 1975, "count": 6 }, { "year": 1976, "count": 5 }, { "year": 1977, "count": 9 }, { "year": 1978, "count": 16 }, { "year": 1979, "count": 31 }, { "year": 1980, "count": 49 }, { "year": 1981, "count": 58 }, { "year": 1982, "count": 29 }, { "year": 1983, "count": 27 }, { "year": 1984, "count": 27 }, { "year": 1985, "count": 28 }, { "year": 1986, "count": 21 }, { "year": 1987, "count": 40 }, { "year": 1988, "count": 24 }, { "year": 1989, "count": 34 }, { "year": 1990, "count": 34 }, { "year": 1991, "count": 33 }, { "year": 1992, "count": 30 }, { "year": 1993, "count": 35 }, { "year": 1994, "count": 42 }, { "year": 1995, "count": 39 }, { "year": 1996, "count": 55 }, { "year": 1997, "count": 59 }, { "year": 1998, "count": 64 }, { "year": 1999, "count": 87 }, { "year": 2000, "count": 177 }, { "year": 2001, "count": 155 }, { "year": 2002, "count": 130 }, { "year": 2003, "count": 190 }, { "year": 2004, "count": 285 }, { "year": 2005, "count": 463 }, { "year": 2006, "count": 1344 }, { "year": 2007, "count": 66280 }, { "year": 2008, "count": 67818 }, { "year": 2009, "count": 66273 }, { "year": 2010, "count": 70144 }, { "year": 2011, "count": 72870 }, { "year": 2012, "count": 67273 }, { "year": 2013, "count": 63118 }, { "year": 2014, "count": 64247 }, { "year": 2015, "count": 66779 }, { "year": 2016, "count": 69726 }, { "year": 2017, "count": 66608 }, { "year": 2018, "count": 64375 }, { "year": 2019, "count": 59576 }, { "year": 2020, "count": 58035 }, { "year": 2021, "count": 58596 }, { "year": 2022, "count": 57885 }, { "year": 2023, "count": 59466 }, { "year": 2024, "count": 57757 }, { "year": 2025, "count": 56278 }, { "year": 2026, "count": 26010 }, { "year": 2030, "count": 1 }, { "year": 2055, "count": 1 }, { "year": 2057, "count": 1 }, { "year": 2060, "count": 1 }, { "year": 2062, "count": 1 }, { "year": 2063, "count": 1 }, { "year": 2064, "count": 1 }, { "year": 2065, "count": 3 }, { "year": 2066, "count": 1 }, { "year": 2067, "count": 3 }, { "year": 2068, "count": 2 } ], "enum_distributions": [ { "field": "offense_type", "distinct_values": 60, "null_count": 0, "null_pct": 0.0, "top_values": [ { "value": "other", "count": 136128, "pct": 11.0 }, { "value": "other_theft", "count": 130296, "pct": 10.5 }, { "value": "property_destruction", "count": 103265, "pct": 8.3 }, { "value": "burglary", "count": 101675, "pct": 8.2 }, { "value": "simple_assault", "count": 93532, "pct": 7.5 }, { "value": "theft_from_vehicle", "count": 84616, "pct": 6.8 }, { "value": "drug_manufacturing", "count": 82870, "pct": 6.7 }, { "value": "shoplifting", "count": 75238, "pct": 6.1 }, { "value": "motor_vehicle_theft", "count": 65599, "pct": 5.3 }, { "value": "unknown_unmapped", "count": 46973, "pct": 3.8 }, { "value": "intimidation", "count": 44714, "pct": 3.6 }, { "value": "drug_possession", "count": 37066, "pct": 3.0 }, { "value": "aggravated_assault", "count": 36930, "pct": 3.0 }, { "value": "theft_from_building", "count": 28812, "pct": 2.3 }, { "value": "robbery", "count": 23917, "pct": 1.9 }, { "value": "theft_of_vehicle_parts", "count": 22959, "pct": 1.8 }, { "value": "family_offense", "count": 22840, "pct": 1.8 }, { "value": "credit_card_fraud", "count": 21775, "pct": 1.8 }, { "value": "identity_theft", "count": 13718, "pct": 1.1 }, { "value": "weapon_violation", "count": 12008, "pct": 1.0 }, { "value": "stolen_property", "count": 11245, "pct": 0.9 }, { "value": "counterfeiting", "count": 10318, "pct": 0.8 }, { "value": "false_pretenses", "count": 5576, "pct": 0.4 }, { "value": "trespass", "count": 4464, "pct": 0.4 }, { "value": "fondling", "count": 3851, "pct": 0.3 } ], "tail_count": 22392, "truncated": true }, { "field": "status", "distinct_values": 2, "null_count": 0, "null_pct": 0.0, "top_values": [ { "value": "completed", "count": 1176433, "pct": 94.7 }, { "value": "attempted", "count": 66344, "pct": 5.3 } ], "tail_count": 0, "truncated": false } ] }, "grain": "One row per reported incident. UNIQUE(source_id, source_dataset); source carries ~1.18 offense-rows per incident, deduped to one row via ON CONFLICT.", "ingest_tool": "civic-data ingest crime", "overlay": { "measures_vs_assumes": "Measures crimes REPORTED to and recorded by LMPD — not crime. Under-reported offense types (much property/sexual crime) are structurally absent; enforcement intensity shapes the counts.", "representativeness_bias": "HIGHEST-STAKES field. Over-policed areas read as 'high crime' when they may be high-enforcement. Presenting reported-crime density as neutral 'safety' risks amplifying policing bias. Frame as reported incidents, never 'how dangerous a place is'.", "pii_tier": "sensitive-aggregate", "license": "Louisville Metro Open Data — published under the Open Data Commons Public Domain Dedication and License (PDDL; verified 2026-07-11 at data.louisvilleky.gov/pages/terms-of-use-and-license, backed by Open Data Ordinance O-243-22). Share/create/adapt permitted, no attribution required; when republishing modified data, identify the source, version, and modifications made - which this project's generated lineage layer does per dataset. We license only our added value (normalization + documentation), never the city's underlying data - see LICENSE-data (CC BY 4.0).", "trend_window": "2007–present for incident counts (NIBRS-era services). Occurrence dates BEFORE 2007 appear on later-reported incidents (cold cases/delayed reports) and future-dated typos exist — filter by incident_date range before trending.", "definitional_drift": "NIBRS coverage shifts 2007–2011: unmapped codes 6–7.5%/yr then 1–3% from 2012. Violent/property split slightly understated in 2007–2011. Offense-dedup keeps one offense per incident (last-write-wins), so the offense MIX can be marginally off; incident COUNTS are sound.", "as_of": "Per-year services 2007–2026 (frozen historical layers).", "cadence": "Source updates continuously; per-year services re-pulled on demand.", "provenance": "ArcGIS Feature Services, one service per year (CRIME_SERVICES registry)." }, "lineage": { "kind": "transactional", "source_system": "Louisville Metro / LOJIC ArcGIS Hub — per-year 'Crime Data' Feature Services (CRIME_SERVICES registry, one frozen service per year 2007–2026).", "silver_table": "silver.crime_incidents", "why_summary": "Turn 17 years of drifting per-year LMPD exports into one typed, NIBRS-categorized, geography-validated incident table — while being explicit that it measures REPORTED crime (enforcement-shaped), not crime, and that location-less offenses are dropped.", "transforms": [ { "op": "fetch", "detail": "Paginated ArcGIS query per year (resultOffset/exceededTransferLimit).", "why": "One frozen service per year is the reproducibility anchor — a fixed source to re-pull.", "loss": "Fetch-stage gaps are invisible unless bronze raw is persisted; the source-count audit bounds this from the source side." }, { "op": "normalize_fields", "detail": "Three field-naming eras unified to canonical lowercase names: 2007–2019 UPPERCASE + string dates, 2020–2023 UPPERCASE + ISO dates, 2024–2026 lowercase + epoch-ms dates.", "why": "One silver schema across 17 years of source schema drift.", "loss": "None — pure renaming/aliasing." }, { "op": "parse", "detail": "date_occurred parsed from 5 string formats or epoch-ms → DATE + TIME.", "why": "A typed incident_date is the temporal grain for every downstream count.", "loss": "Records whose date cannot be parsed (or is absent) are SKIPPED — no incident_date, no row." }, { "op": "map_enum", "detail": "NIBRS code → offense_type via NIBRS_CATEGORIES; unmapped-but-present codes → 'unknown_unmapped'; absent code → 'unknown_true'.", "why": "Human-readable offense categories (violent/property/other) instead of raw NIBRS codes.", "loss": "Offense granularity collapses to category; 2007–2011 carries more unmapped codes (6–7.5%/yr), so the violent/property split is marginally understated in that era." }, { "op": "validate_zip", "detail": "ZIP validated against the canonical silver.zip_codes MDM set (three-state: valid → keep normalized; missing → NULL; non-Louisville → quarantine).", "why": "One canonical geography; no record silently lands on an out-of-area or invented ZIP.", "loss": "Missing ZIP kept as NULL (unknown, not dropped)." }, { "op": "quarantine", "detail": "Location-less offenses (fraud, identity theft, cyber) stamped by LMPD with placeholder ZIP 40056 fall outside the set → quarantined to silver.quarantine.", "why": "They have no physical location and cannot be placed; keeping them would invent geography.", "loss": "CITYWIDE crime undercounts by the quarantined share, biased toward non-geographic offense types. Per-ZIP crime is unaffected. This asymmetry vs. 311 is a known open question.", "live": { "quarantined_at_40056": 19734, "crime_silver": 1242777, "crime_quarantined_total": 20484, "citywide_undercount_pct": 1.6 } }, { "op": "dedup", "detail": "ON CONFLICT(source_id, source_dataset) last-write-wins; source carries ~1.18 offense-rows per incident, collapsed to one incident row.", "why": "Incident grain — one row per reported incident, not per offense charge.", "loss": "The offense MIX can be marginally off (last-write-wins picks one offense); incident COUNTS are sound." }, { "op": "geocode", "detail": "Block/intersection addresses matched to the LOJIC reference tables (silver.address_points ~450K / silver.street_intersections ~21K, re-ingested from source by `ingest-refs`) → silver.crime_geocode, applied back to crime.geom (block-centroid / intersection point). ~97% of addressed crime placed.", "why": "Crime carries NO coordinates at source; a point enables sub-ZIP placement and district joins.", "loss": "~2.8% of addressed crime matches neither form (highway '@...', 'COMMUNITY AT LARGE', odd formats) and stays ZIP-level only (geom NULL). The reference tables are now reproduced from source, so this is a residual match-rate limit, not a reproduction gap." }, { "op": "derive", "detail": "Where the source ZIP is absent, the geocoded point is placed into the canonical silver.zip_codes boundary (point-in-polygon) to fill zip_code, flagged zip_derived=TRUE.", "why": "Recovers ZIP for ~1,060 otherwise-unplaced records without trusting the source's own ZIP field.", "loss": "Fills only where a geocoded point exists AND falls inside a boundary; edge/unmatched points stay NULL. zip_derived marks the derivation so consumers can separate it from source ZIP." }, { "op": "strip_pii", "detail": "No person-name fields are written; crime quarantine JSONB is de-identified.", "why": "No-PII-in-DB policy — private individuals' names never persist.", "loss": "None relevant to analysis (names are not an analytical field); addresses retained." } ], "produces_columns": [ "source_id", "incident_date", "incident_time", "offense_type", "offense_description", "location_address", "zip_code", "zip_derived", "premise_type", "status", "geom", "source_dataset", "bronze_id" ], "reproduce": { "commands": [ "civic-data ingest-zips # load the canonical ZIP MDM first", "civic-data ingest-refs # load LOJIC geocoding refs (address points + intersections)", "civic-data ingest crime --all --promote # source → bronze → silver, all years", "civic-data geocode-crime --write --apply # place crime: geom + point-in-polygon derived ZIP" ], "gaps": [ "~2.8% of addressed crime matches neither the block nor intersection form (highway '@...', 'COMMUNITY AT LARGE', non-standard strings) and remains ZIP-level only (geom NULL) — a residual match-rate limit. The LOJIC reference tables ARE now reproduced from source (ingest-refs), so the former local-only reproduction gap is closed." ] }, "verified_against_schema": true, "note": null, "quarantine_live": 20484, "source_vs_silver": { "source": null, "silver": 1242777, "quarantine": 20484, "gap": null, "gap_pct": null } } }, { "key": "service_requests", "title": "311 service requests (Open311)", "trust_tier": "viewable_caveated", "generated": { "schema": "silver", "table": "service_requests", "row_count": 2209988, "primary_key": [ "id" ], "year_column": "requested_datetime", "fields": [ { "name": "id", "type": "bigint", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "service_request_id", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "requested_datetime", "type": "timestamp with time zone", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "updated_datetime", "type": "timestamp with time zone", "nullable": true, "null_count": 1668718, "null_pct": 75.5 }, { "name": "closed_datetime", "type": "timestamp with time zone", "nullable": true, "null_count": 1716678, "null_pct": 77.7 }, { "name": "status", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "service_name", "type": "text", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "service_code", "type": "text", "nullable": true, "null_count": 2209988, "null_pct": 100.0 }, { "name": "description", "type": "text", "nullable": true, "null_count": 639533, "null_pct": 28.9 }, { "name": "agency_responsible", "type": "text", "nullable": true, "null_count": 1238235, "null_pct": 56.0 }, { "name": "address", "type": "text", "nullable": true, "null_count": 583416, "null_pct": 26.4 }, { "name": "latitude", "type": "double precision", "nullable": true, "null_count": 523647, "null_pct": 23.7 }, { "name": "longitude", "type": "double precision", "nullable": true, "null_count": 523647, "null_pct": 23.7 }, { "name": "geom", "type": "USER-DEFINED", "nullable": true }, { "name": "census_tract_geoid", "type": "text", "nullable": true, "null_count": 2209988, "null_pct": 100.0 }, { "name": "neighborhood_id", "type": "integer", "nullable": true, "null_count": 2209988, "null_pct": 100.0 }, { "name": "zip_code", "type": "text", "nullable": true, "null_count": 844743, "null_pct": 38.2 }, { "name": "resolution_days", "type": "integer", "nullable": true, "null_count": 1716678, "null_pct": 77.7 }, { "name": "source_dataset", "type": "text", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "bronze_id", "type": "bigint", "nullable": true, "null_count": 0, "null_pct": 0.0 } ], "per_year": [ { "year": 2010, "count": 85593 }, { "year": 2011, "count": 86921 }, { "year": 2012, "count": 85340 }, { "year": 2013, "count": 79816 }, { "year": 2014, "count": 84929 }, { "year": 2015, "count": 92838 }, { "year": 2016, "count": 93659 }, { "year": 2017, "count": 94113 }, { "year": 2018, "count": 110907 }, { "year": 2019, "count": 146953 }, { "year": 2020, "count": 229875 }, { "year": 2021, "count": 216399 }, { "year": 2022, "count": 175150 }, { "year": 2023, "count": 173244 }, { "year": 2024, "count": 169588 }, { "year": 2025, "count": 182486 }, { "year": 2026, "count": 102177 } ], "enum_distributions": [ { "field": "service_name", "distinct_values": 370, "null_count": 0, "null_pct": 0.0, "top_values": [ { "value": "NSR Metro Agencies", "count": 296006, "pct": 13.4 }, { "value": "Streets", "count": 97481, "pct": 4.4 }, { "value": "Large Item Appointment", "count": 81394, "pct": 3.7 }, { "value": "Solid Waste Container Request", "count": 76464, "pct": 3.5 }, { "value": "Solid Waste Missed Services", "count": 73693, "pct": 3.3 }, { "value": "NSR Social Services", "count": 65553, "pct": 3.0 }, { "value": "HIGH GRASS WEEDS", "count": 65276, "pct": 3.0 }, { "value": "Exterior", "count": 63325, "pct": 2.9 }, { "value": "CRM", "count": 60093, "pct": 2.7 }, { "value": "STREET POTHOLE", "count": 48980, "pct": 2.2 }, { "value": "Parking Concern", "count": 48556, "pct": 2.2 }, { "value": "EXTERIOR VIOL", "count": 44630, "pct": 2.0 }, { "value": "NSR Miscellaneous", "count": 43058, "pct": 1.9 }, { "value": "Animal Issue", "count": 41157, "pct": 1.9 }, { "value": "High Weeds/Grass", "count": 40293, "pct": 1.8 }, { "value": "TRASH PVT PROP", "count": 38634, "pct": 1.7 }, { "value": "DAMAGE GARB CART", "count": 34663, "pct": 1.6 }, { "value": "NSR Government", "count": 34454, "pct": 1.6 }, { "value": "GARBAGE MISSED", "count": 32609, "pct": 1.5 }, { "value": "RECYCLE BINS", "count": 28470, "pct": 1.3 }, { "value": "MISCELLANEOUS", "count": 27560, "pct": 1.2 }, { "value": "Metro Agencies", "count": 27381, "pct": 1.2 }, { "value": "ABANDON VEHICLE", "count": 25486, "pct": 1.2 }, { "value": "DEAD ANIMAL", "count": 25187, "pct": 1.1 }, { "value": "Trash", "count": 24551, "pct": 1.1 } ], "tail_count": 765034, "truncated": true }, { "field": "status", "distinct_values": 2, "null_count": 0, "null_pct": 0.0, "top_values": [ { "value": "closed", "count": 1420000, "pct": 64.3 }, { "value": "open", "count": 789988, "pct": 35.7 } ], "tail_count": 0, "truncated": false }, { "field": "agency_responsible", "distinct_values": 238, "null_count": 1238235, "null_pct": 56.0, "top_values": [ { "value": "IPL-PROPERTY MAINTENANCE DIV.", "count": 238342, "pct": 10.8 }, { "value": "SWMS - ADMINISTRATION", "count": 117511, "pct": 5.3 }, { "value": "LPM", "count": 94771, "pct": 4.3 }, { "value": "SWMS - WASTE COLLECTION", "count": 92084, "pct": 4.2 }, { "value": "SWMS - ENFORCEMENT UNIT", "count": 45486, "pct": 2.1 }, { "value": "METRO POLICE DEPARTMENT", "count": 41057, "pct": 1.9 }, { "value": "LMPD", "count": 30499, "pct": 1.4 }, { "value": "Public Works Engineering Div.", "count": 20952, "pct": 0.9 }, { "value": "PUBLIC WORKS ADMINISTRATION", "count": 18318, "pct": 0.8 }, { "value": "PUBLIC WORKS OPEN SPACES DIV", "count": 17931, "pct": 0.8 }, { "value": "Louisville Animal Services", "count": 16588, "pct": 0.8 }, { "value": "SWMS - BULK WASTE", "count": 15412, "pct": 0.7 }, { "value": "C&R Property Maintenance Enforcement", "count": 15128, "pct": 0.7 }, { "value": "PUBLIC WORKS - SIGNS MARKINGS", "count": 13470, "pct": 0.6 }, { "value": "PUBLIC WORKS INSPECTION GROUP", "count": 13261, "pct": 0.6 }, { "value": "HEALTH DEPT - ENVIR. HEALTH", "count": 12318, "pct": 0.6 }, { "value": "ANIMAL SERVICES", "count": 11799, "pct": 0.5 }, { "value": "PUBLIC WORKS - TRAFFIC ENG", "count": 11073, "pct": 0.5 }, { "value": "PUBLIC WORKS - ELEC MAINT", "count": 10939, "pct": 0.5 }, { "value": "METRO PARKS", "count": 10206, "pct": 0.5 }, { "value": "Parking Authority River City", "count": 8419, "pct": 0.4 }, { "value": "KY DEPT OF TRANSPORTATION", "count": 8307, "pct": 0.4 }, { "value": "Public Works Electrical Maint", "count": 7629, "pct": 0.3 }, { "value": "PUBLIC WORKS ELECTRICAL MAINT.", "count": 7002, "pct": 0.3 }, { "value": "N-PICKUP DAYS", "count": 6954, "pct": 0.3 } ], "tail_count": 86297, "truncated": true } ] }, "grain": "One row per service request. UNIQUE(service_request_id, source_dataset).", "ingest_tool": "civic-data ingest 311", "overlay": { "measures_vs_assumes": "Measures who CALLS 311 (and which agency logs it) — not where problems are. A non-report is invisible; absence ≠ no problem.", "representativeness_bias": "Civically-engaged / higher-trust areas over-report; under-served areas under-report the same conditions. Reading request density as 'problem density' inverts the bias.", "pii_tier": "public-aggregate-safe", "license": "Louisville Metro Open Data — published under the Open Data Commons Public Domain Dedication and License (PDDL; verified 2026-07-11 at data.louisvilleky.gov/pages/terms-of-use-and-license, backed by Open Data Ordinance O-243-22). Share/create/adapt permitted, no attribution required; when republishing modified data, identify the source, version, and modifications made - which this project's generated lineage layer does per dataset. We license only our added value (normalization + documentation), never the city's underlying data - see LICENSE-data (CC BY 4.0).", "trend_window": "2010–2018 is the honest trend window (canonical home of this claim; location quality degrades after 2018 — see definitional_drift).", "definitional_drift": "'other' category bucket ~66% (2010) → ~32% (2025) as service names shifted. LOCATION QUALITY DEGRADED AFTER 2018: from 2019 on the source stamps a growing share of requests with one placeholder coordinate (downtown) carrying no usable ZIP — hitting REAL categories (Streets, Traffic Signals), not just admin. 2020–2021 have no location at all. resolution_days NULL for all pre-2022 data. 2007–2009 excluded.", "as_of": "Per-year services 2010–2026 (2019 split into two half-year services).", "cadence": "Continuous at source; re-pulled per service on demand.", "provenance": "ArcGIS Feature Services (SR_SERVICES registry). A large share of records carry NULL ZIP — kept as NULL (recoverable via lat/lng), never quarantined." }, "lineage": { "kind": "transactional", "source_system": "Louisville Metro / LOJIC ArcGIS Hub — per-year '311 Service Requests' Feature Services (SR_SERVICES registry; 2019 split into two half-year services; 2007–2009 excluded).", "silver_table": "silver.service_requests", "why_summary": "Unify Open311 exports across schema eras into one typed request table with computed resolution time and a point geometry — keeping missing-location records (unlike crime) and being explicit that it measures reporting propensity, not problem density.", "transforms": [ { "op": "fetch", "detail": "Paginated ArcGIS query per year/half-year.", "why": "Frozen per-year services are the reproducibility anchor.", "loss": "Fetch-stage gaps invisible without persisted bronze; source-count audit bounds it." }, { "op": "normalize_fields", "detail": "Era differences reconciled: status↔status_description, closed_datetime↔closed_date, lat/long↔latitude/longitude, and the distinct 2020–2021 'Record__/Opened_Date' schema mapped to canonical names.", "why": "One silver schema across Open311 source drift.", "loss": "None — renaming/aliasing." }, { "op": "derive", "detail": "When zip_code is absent, extract a trailing 5-digit ZIP from the address string (e.g. '… LOUISVILLE, KY, 40229').", "why": "Best-effort ZIP recovery for older eras that embedded ZIP in the address.", "loss": "None added; a recovery, not a drop. Unrecoverable → left for validate_zip to NULL." }, { "op": "parse", "detail": "requested/closed/updated parsed from epoch-ms → TIMESTAMPTZ.", "why": "Typed timestamps are the temporal grain and the input to resolution time.", "loss": "Records with no parseable requested_datetime are SKIPPED." }, { "op": "derive", "detail": "resolution_days = max(0, closed − requested) in days.", "why": "A normalized resolution measure independent of source formatting.", "loss": "NULL whenever closed is absent — all pre-2022 data has NULL resolution_days." }, { "op": "build_geom", "detail": "ST_MakePoint(lng, lat) → geom when both coordinates present.", "why": "Point geometry enables spatial joins and sub-ZIP placement.", "loss": "NULL geom when coordinates are absent — a growing share post-2018 (placeholder coordinate)." }, { "op": "validate_zip", "detail": "ZIP validated against the canonical silver.zip_codes MDM set (three-state).", "why": "One canonical geography.", "loss": "Missing ZIP KEPT as NULL (recoverable via lat/lng); only non-Louisville ZIPs quarantined. This differs from crime, which quarantines its location-less records — a documented asymmetry." }, { "op": "derive", "detail": "Require a non-empty service_name; drop the record otherwise.", "why": "service_name is NOT NULL in silver — a nameless request has no analytical category.", "loss": "Records with empty service_name are SKIPPED." }, { "op": "dedup", "detail": "ON CONFLICT(service_request_id, source_dataset) upsert.", "why": "One row per service request per source dataset.", "loss": "None beyond de-duplication of re-pulled records." } ], "produces_columns": [ "service_request_id", "requested_datetime", "updated_datetime", "closed_datetime", "status", "service_name", "description", "agency_responsible", "address", "latitude", "longitude", "geom", "zip_code", "resolution_days", "source_dataset", "bronze_id" ], "reproduce": { "commands": [ "civic-data ingest-zips # canonical ZIP MDM first", "civic-data ingest 311 --all --promote # source → bronze → silver, all years" ], "gaps": [ "Post-2018 location degradation is a SOURCE condition, faithfully preserved (NULL geom / NULL ZIP kept), not something the recipe can fix. Trend use is honest only within 2010–2018." ] }, "verified_against_schema": true, "note": null, "quarantine_live": 178, "source_vs_silver": { "source": null, "silver": 2209988, "quarantine": 178, "gap": null, "gap_pct": null } } }, { "key": "landbank_sales", "title": "Land Bank Authority sales (historical dispositions)", "trust_tier": "viewable_caveated", "generated": { "schema": "silver", "table": "landbank_sales", "row_count": 399, "primary_key": [ "id" ], "year_column": "sale_date", "fields": [ { "name": "id", "type": "bigint", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "parcel_id", "type": "text", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "address", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "zip_code", "type": "text", "nullable": true, "null_count": 9, "null_pct": 2.3 }, { "name": "neighborhood", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "council_district", "type": "integer", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "zoning", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "has_improvements", "type": "boolean", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "sale_program", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "sale_date", "type": "date", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "sale_amount", "type": "text", "nullable": true, "null_count": 1, "null_pct": 0.3 }, { "name": "source_dataset", "type": "text", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "bronze_id", "type": "bigint", "nullable": true, "null_count": 0, "null_pct": 0.0 } ], "per_year": [ { "year": 2015, "count": 21 }, { "year": 2016, "count": 27 }, { "year": 2017, "count": 82 }, { "year": 2018, "count": 99 }, { "year": 2019, "count": 97 }, { "year": 2020, "count": 73 } ], "enum_distributions": [ { "field": "sale_program", "distinct_values": 10, "null_count": 0, "null_pct": 0.0, "top_values": [ { "value": "Cut It Keep It", "count": 75, "pct": 18.8 }, { "value": "Save the Structure", "count": 74, "pct": 18.5 }, { "value": "Side Yard", "count": 67, "pct": 16.8 }, { "value": "RFP", "count": 64, "pct": 16.0 }, { "value": "Budget Rate", "count": 43, "pct": 10.8 }, { "value": "Demo for Deed", "count": 21, "pct": 5.3 }, { "value": "Minimum Pricing Policy", "count": 21, "pct": 5.3 }, { "value": "Flex Rate", "count": 20, "pct": 5.0 }, { "value": "SOI", "count": 9, "pct": 2.3 }, { "value": "Metro Redevelopment", "count": 5, "pct": 1.3 } ], "tail_count": 0, "truncated": false }, { "field": "zoning", "distinct_values": 19, "null_count": 0, "null_pct": 0.0, "top_values": [ { "value": "R6", "count": 155, "pct": 38.8 }, { "value": "R5", "count": 71, "pct": 17.8 }, { "value": "UN", "count": 49, "pct": 12.3 }, { "value": "R7", "count": 37, "pct": 9.3 }, { "value": "EZ1", "count": 22, "pct": 5.5 }, { "value": "C1", "count": 21, "pct": 5.3 }, { "value": "OR2", "count": 15, "pct": 3.8 }, { "value": "C2", "count": 11, "pct": 2.8 }, { "value": "CR", "count": 4, "pct": 1.0 }, { "value": "TNZD", "count": 3, "pct": 0.8 }, { "value": "M2", "count": 2, "pct": 0.5 }, { "value": "R4", "count": 2, "pct": 0.5 }, { "value": "R8A", "count": 1, "pct": 0.3 }, { "value": "M3", "count": 1, "pct": 0.3 }, { "value": "R8", "count": 1, "pct": 0.3 }, { "value": "EZ1 ", "count": 1, "pct": 0.3 }, { "value": "CM", "count": 1, "pct": 0.3 }, { "value": "C3", "count": 1, "pct": 0.3 }, { "value": "RSA", "count": 1, "pct": 0.3 } ], "tail_count": 0, "truncated": false } ] }, "grain": "One row per disposed property. UNIQUE(parcel_id, source_dataset).", "ingest_tool": "civic-data ingest landbank", "overlay": { "measures_vs_assumes": "Measures completed Land Bank Authority DISPOSITIONS — vacant/abandoned parcels the city's land bank actually SOLD — not the distressed-property inventory nor demand. A row is a successful sale; absence means not-yet-sold or never-in-inventory.", "representativeness_bias": "Concentrated in historically disinvested neighborhoods where the land bank operates. Not a citywide property-sales measure; reading disposition counts as 'investment' conflates public disposal of blight with private market activity.", "pii_tier": "public-aggregate-safe", "license": "Louisville Metro Open Data — published under the Open Data Commons Public Domain Dedication and License (PDDL; verified 2026-07-11 at data.louisvilleky.gov/pages/terms-of-use-and-license, backed by Open Data Ordinance O-243-22). Share/create/adapt permitted, no attribution required; when republishing modified data, identify the source, version, and modifications made - which this project's generated lineage layer does per dataset. We license only our added value (normalization + documentation), never the city's underlying data - see LICENSE-data (CC BY 4.0).", "definitional_drift": "The 'Historical Data' service — a completed-sales log, not a live inventory. Small N (hundreds). sale_program categories shift over the program's life; sale_amount is a raw source string (not coerced to numeric).", "as_of": "Single pull (historical service).", "cadence": "Source updates as dispositions complete; re-pulled on demand.", "provenance": "ArcGIS Feature Service — Landbank_Sales_Historical_Data (single pull). Owner/applicant names stripped before any DB write (no-PII policy)." }, "lineage": { "kind": "transactional", "source_system": "Louisville Metro / LOJIC ArcGIS Hub — 'Landbank_Sales_Historical_Data' Feature Service (single pull; a completed-dispositions log).", "silver_table": "silver.landbank_sales", "why_summary": "Turn the Land Bank Authority's historical disposition export into one typed, ZIP-validated sales log — being explicit that it measures completed PUBLIC dispositions of blighted parcels (small N), not distressed inventory or market activity, and that owner/applicant names never persist.", "transforms": [ { "op": "fetch", "detail": "Single ArcGIS query (historical service, hundreds of rows).", "why": "The historical service is a fixed reproducibility anchor — one pull, fully re-runnable.", "loss": "Fetch-stage gaps are invisible unless bronze raw is persisted (bounded by bronze)." }, { "op": "strip_pii", "detail": "OWNER + APPLICANT dropped before any DB write.", "why": "No-PII-in-DB policy — private individuals' names never persist.", "loss": "None relevant to analysis; the property address is retained." }, { "op": "normalize_fields", "detail": "address assembled from ST_/DIR/STREET_NAME/ST_TYPE/Post_Dir; council_district coerced from COUNCIL_DIST → int; has_improvements from IMPROV → bool.", "why": "One typed silver schema from the source's split address + string fields.", "loss": "None — assembling/typing. sale_amount kept as the raw source string (not coerced numeric)." }, { "op": "validate_zip", "detail": "ZIP validated against the canonical silver.zip_codes MDM set (three-state: valid → keep; missing → NULL; non-Louisville → quarantine).", "why": "One canonical geography; no record lands on an out-of-area or invented ZIP.", "loss": "Missing ZIP kept as NULL (unknown, not dropped)." }, { "op": "dedup", "detail": "ON CONFLICT(parcel_id, source_dataset) upsert.", "why": "One row per disposed parcel.", "loss": "None beyond de-duplication of re-pulled records." } ], "produces_columns": [ "parcel_id", "address", "zip_code", "neighborhood", "council_district", "zoning", "has_improvements", "sale_program", "sale_date", "sale_amount", "source_dataset", "bronze_id" ], "reproduce": { "commands": [ "civic-data ingest-zips # canonical ZIP MDM first", "civic-data ingest landbank --promote # source → bronze → silver (single pull)" ], "gaps": [ "Reproduces civic-graph EXACTLY — content hash identical (399 = 399 rows). Historical service only; there is no live-inventory feed, so this is a completed-sales log, not current holdings." ] }, "verified_against_schema": true, "note": null, "quarantine_live": 0 } }, { "key": "lien_orders", "title": "Code-enforcement lien final orders (blight signal)", "trust_tier": "snapshot_only", "generated": { "schema": "silver", "table": "lien_orders", "row_count": 298, "primary_key": [ "id" ], "year_column": "notification_date", "fields": [ { "name": "id", "type": "bigint", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "case_id", "type": "text", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "property_address", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "zip_code", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "latitude", "type": "double precision", "nullable": true, "null_count": 1, "null_pct": 0.3 }, { "name": "longitude", "type": "double precision", "nullable": true, "null_count": 1, "null_pct": 0.3 }, { "name": "geom", "type": "USER-DEFINED", "nullable": true }, { "name": "notification_date", "type": "date", "nullable": true, "null_count": 45, "null_pct": 15.1 }, { "name": "inspection_date", "type": "date", "nullable": true, "null_count": 45, "null_pct": 15.1 }, { "name": "inspection_result", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "hearing_scheduled", "type": "date", "nullable": true, "null_count": 280, "null_pct": 94.0 }, { "name": "hearing_completed", "type": "date", "nullable": true, "null_count": 286, "null_pct": 96.0 }, { "name": "hearing_result", "type": "text", "nullable": true, "null_count": 286, "null_pct": 96.0 }, { "name": "final_order_state", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "citation_amount", "type": "double precision", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "source_dataset", "type": "text", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "bronze_id", "type": "bigint", "nullable": true, "null_count": 0, "null_pct": 0.0 } ], "per_year": [ { "year": 2025, "count": 2 }, { "year": 2026, "count": 251 } ], "enum_distributions": [ { "field": "final_order_state", "distinct_values": 3, "null_count": 0, "null_pct": 0.0, "top_values": [ { "value": "Final By Citation", "count": 281, "pct": 94.3 }, { "value": "Final By Hearing", "count": 12, "pct": 4.0 }, { "value": "Rescinded", "count": 5, "pct": 1.7 } ], "tail_count": 0, "truncated": false }, { "field": "inspection_result", "distinct_values": 7, "null_count": 0, "null_pct": 0.0, "top_values": [ { "value": "Citation", "count": 168, "pct": 56.4 }, { "value": "Citation Referral", "count": 63, "pct": 21.1 }, { "value": "Cancelled", "count": 45, "pct": 15.1 }, { "value": "Emergency Referral", "count": 16, "pct": 5.4 }, { "value": "Corrective Order Citation", "count": 4, "pct": 1.3 }, { "value": "Hold", "count": 1, "pct": 0.3 }, { "value": "Violation Notice", "count": 1, "pct": 0.3 } ], "tail_count": 0, "truncated": false } ] }, "grain": "One row per case. UNIQUE(case_id, source_dataset). Geocoded at source (lat/lng → geom).", "ingest_tool": "civic-data ingest liens", "overlay": { "measures_vs_assumes": "Measures code-enforcement lien FINAL ORDERS currently active at the source (an enforcement/blight signal), not total historical liens nor total blight. A non-finalized or resolved case is invisible.", "representativeness_bias": "Enforcement-driven: reflects where the city has inspected AND finalized liens, shaped by inspection intensity — not raw property condition. High-enforcement areas read as high-lien.", "pii_tier": "public-aggregate-safe", "license": "Louisville Metro Open Data — published under the Open Data Commons Public Domain Dedication and License (PDDL; verified 2026-07-11 at data.louisvilleky.gov/pages/terms-of-use-and-license, backed by Open Data Ordinance O-243-22). Share/create/adapt permitted, no attribution required; when republishing modified data, identify the source, version, and modifications made - which this project's generated lineage layer does per dataset. We license only our added value (normalization + documentation), never the city's underlying data - see LICENSE-data (CC BY 4.0).", "definitional_drift": "SOURCE VOLATILITY (the defining property): the feed is a ROLLING ~7–8-month window of active final orders — resolved cases drop off as new ones appear. A single from-source pull is a point-in-time SNAPSHOT (298 records, notification dates 2025-11→2026-06); civic-graph's 798 is an ACCUMULATION across many re-pulls over months (same case-id format, near-zero overlap with today's window). The two are not comparable as a time series and the store cannot be reproduced from one pull — snapshot only.", "as_of": "Single snapshot pull (2026-07).", "cadence": "Source churns continuously; resolved cases leave the window.", "provenance": "ArcGIS Feature Service — lien_notification_final_orders (single pull, geocoded at source). Inspector/owner names stripped before any DB write." }, "lineage": { "kind": "transactional", "source_system": "Louisville Metro / LOJIC ArcGIS Hub — 'lien_notification_final_orders' Feature Service (single pull, geocoded at source).", "silver_table": "silver.lien_orders", "why_summary": "Turn the code-enforcement lien final-orders feed into a typed, geocoded, ZIP-validated case table — while documenting that the source is a ROLLING WINDOW of active orders, so a single from-source pull is a point-in-time SNAPSHOT, not a reproducible historical store.", "transforms": [ { "op": "fetch", "detail": "Single ArcGIS query of the live final-orders feed.", "why": "One pull captures the current active-order window.", "loss": "SOURCE VOLATILITY, not fetch loss: orders RESOLVED before the pull have already left the feed. The feed is a rolling ~7–8-month window (our pull: notification dates 2025-11→2026-06)." }, { "op": "strip_pii", "detail": "inspector + fullname dropped before any DB write.", "why": "No-PII-in-DB policy — inspector and property-owner names never persist.", "loss": "None relevant to analysis; the property address is retained." }, { "op": "normalize_fields", "detail": "address from stno/predir/stname/suffix/stsub; citation_amount from final_citation_amount → float; notification/inspection/hearing dates epoch-ms → DATE.", "why": "One typed silver schema from the source's split address + epoch fields.", "loss": "None — assembling/typing." }, { "op": "build_geom", "detail": "ST_MakePoint(lng, lat) → geom from the source's own coordinates.", "why": "Liens are geocoded AT SOURCE — a point enables sub-ZIP placement and district joins directly.", "loss": "NULL geom only where the source omits coordinates." }, { "op": "validate_zip", "detail": "ZIP validated against the canonical silver.zip_codes MDM set (three-state).", "why": "One canonical geography.", "loss": "Missing ZIP kept as NULL; non-Louisville ZIPs quarantined (the lienholder MAILING ZIP can differ from the property ZIP — those are dropped, not placed on the wrong ZIP)." }, { "op": "dedup", "detail": "ON CONFLICT(case_id, source_dataset) upsert.", "why": "One row per case.", "loss": "None beyond de-duplication of re-pulled records." } ], "produces_columns": [ "case_id", "property_address", "zip_code", "latitude", "longitude", "geom", "notification_date", "inspection_date", "inspection_result", "hearing_scheduled", "hearing_completed", "hearing_result", "final_order_state", "citation_amount", "source_dataset", "bronze_id" ], "reproduce": { "commands": [ "civic-data ingest-zips # canonical ZIP MDM first", "civic-data ingest liens --promote # source → bronze → silver (single snapshot)" ], "gaps": [ "ROLLING-WINDOW SOURCE — the defining reproducibility limit. A from-source pull yields the CURRENT snapshot (~298 records) with a NULL geom of 0% and dates in a ~7-month window; it does NOT reproduce civic-graph's silver (798). The 798 is an ACCUMULATION across many re-pulls over months — same case-id format, near-zero row overlap with today's window (shifted, ~1-month overlap). To build a longitudinal lien record you must ACCUMULATE snapshots over time; no single pull can reconstruct the accumulated store. Trust tier is snapshot_only for exactly this reason." ] }, "verified_against_schema": true, "note": null, "quarantine_live": 136 } }, { "key": "property_foreclosures", "title": "Property foreclosure filings", "trust_tier": "viewable_caveated", "generated": { "schema": "silver", "table": "property_foreclosures", "row_count": 1734, "primary_key": [ "id" ], "year_column": "action_filed", "fields": [ { "name": "id", "type": "bigint", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "address", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "zip_code", "type": "text", "nullable": true, "null_count": 44, "null_pct": 2.5 }, { "name": "neighborhood", "type": "text", "nullable": true, "null_count": 36, "null_pct": 2.1 }, { "name": "census_tract", "type": "text", "nullable": true, "null_count": 2, "null_pct": 0.1 }, { "name": "council_district", "type": "integer", "nullable": true, "null_count": 9, "null_pct": 0.5 }, { "name": "parcel_id", "type": "text", "nullable": true, "null_count": 1734, "null_pct": 100.0 }, { "name": "action_filed", "type": "date", "nullable": true, "null_count": 62, "null_pct": 3.6 }, { "name": "sale_date", "type": "date", "nullable": true, "null_count": 232, "null_pct": 13.4 }, { "name": "sale_price", "type": "text", "nullable": true, "null_count": 1734, "null_pct": 100.0 }, { "name": "is_metro_purchase", "type": "boolean", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "case_number", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "source_dataset", "type": "text", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "bronze_id", "type": "bigint", "nullable": true, "null_count": 0, "null_pct": 0.0 } ], "per_year": [ { "year": 2008, "count": 1 }, { "year": 2012, "count": 10 }, { "year": 2013, "count": 88 }, { "year": 2014, "count": 122 }, { "year": 2015, "count": 102 }, { "year": 2016, "count": 73 }, { "year": 2017, "count": 123 }, { "year": 2018, "count": 162 }, { "year": 2019, "count": 143 }, { "year": 2020, "count": 60 }, { "year": 2021, "count": 72 }, { "year": 2022, "count": 162 }, { "year": 2023, "count": 223 }, { "year": 2024, "count": 155 }, { "year": 2025, "count": 118 }, { "year": 2026, "count": 58 } ], "enum_distributions": [ { "field": "is_metro_purchase", "distinct_values": 2, "null_count": 0, "null_pct": 0.0, "top_values": [ { "value": "true", "count": 972, "pct": 56.1 }, { "value": "false", "count": 762, "pct": 43.9 } ], "tail_count": 0, "truncated": false } ] }, "grain": "One row per case (case_number = Case_). UNIQUE(case_number, source_dataset); source carries ~1.98 rows/case, deduped to one via ON CONFLICT.", "ingest_tool": "civic-data ingest foreclosures", "overlay": { "measures_vs_assumes": "Measures foreclosure FILINGS (action_filed — a case initiated), not completed foreclosures nor actual loss of home. A filing may be dismissed, settled, or withdrawn; a filing is not an outcome.", "representativeness_bias": "Tracks judicial legal action, not neighborhood distress directly — concentrated where lenders pursue judicial foreclosure. Reading filing density as 'distress' conflates the legal process with the underlying condition.", "pii_tier": "public-aggregate-safe", "license": "Louisville Metro Open Data — published under the Open Data Commons Public Domain Dedication and License (PDDL; verified 2026-07-11 at data.louisvilleky.gov/pages/terms-of-use-and-license, backed by Open Data Ordinance O-243-22). Share/create/adapt permitted, no attribution required; when republishing modified data, identify the source, version, and modifications made - which this project's generated lineage layer does per dataset. We license only our added value (normalization + documentation), never the city's underlying data - see LICENSE-data (CC BY 4.0).", "definitional_drift": "NO Great-Recession era — the source is dense only from ~2013, so the 2008–2012 foreclosure wave is absent (do not read the series as the full history). Source carries ~1.98 rows/case, deduped to one. Uniquely carries neighborhood names + census tract IDs. A small set of old-numbered cases (12-CI-…, 13-CI-…) carry NULL action_filed. The source GROWS (new filings accrue) and CHURNS (old cases occasionally drop or re-appear), so totals drift from an earlier pull while shared historical years match row-for-row. The source DECLARES a parcel-ID field (Full_Parcel_ID) but ships it EMPTY, so parcel_id is all-NULL — parcel-level joins are impossible; join on address/geography instead.", "as_of": "Single pull.", "cadence": "New filings accrue continuously; re-pulled on demand.", "provenance": "ArcGIS Feature Service — Louisville_Metro_KY_Property_Foreclosures (single pull). Purchaser/case-style names stripped before any DB write; only a derived is_metro_purchase boolean kept." }, "lineage": { "kind": "transactional", "source_system": "Louisville Metro / LOJIC ArcGIS Hub — 'Louisville_Metro_KY_Property_Foreclosures' Feature Service (single pull).", "silver_table": "silver.property_foreclosures", "why_summary": "Turn the county foreclosure-filings export into one typed, deduped, ZIP-validated case table carrying neighborhood + census tract — measuring FILINGS (a case initiated), not completed loss of home, dense only from ~2013, with purchaser names replaced by a derived metro-buyer boolean.", "transforms": [ { "op": "fetch", "detail": "Single ArcGIS query.", "why": "One pull is the reproducibility anchor.", "loss": "Fetch-stage gaps bounded by bronze; the source itself grows (new filings) and churns (old cases occasionally drop/re-appear), so a later pull differs from an earlier one." }, { "op": "derive", "detail": "_is_metro_purchase = 'METRO' in Purchaser, computed BEFORE PII stripping.", "why": "Keep the analytical signal (was the buyer Metro/Landbank?) without keeping the name.", "loss": "The specific purchaser identity is intentionally not retained — only the boolean survives." }, { "op": "strip_pii", "detail": "Purchaser + Case_Style dropped before any DB write.", "why": "No-PII-in-DB policy — private buyers' names never persist.", "loss": "None relevant to analysis; the property address is retained." }, { "op": "normalize_fields", "detail": "address from House_Nr/Dir/Street_Name/St_Type/Post_Dir; council_district from CD → int; census_tract from Census_Tract; action_filed/sale_date epoch-ms → DATE.", "why": "One typed silver schema from the source's split address + epoch fields.", "loss": "None — assembling/typing. sale_price kept as the raw source string (not coerced numeric)." }, { "op": "validate_zip", "detail": "ZIP validated against the canonical silver.zip_codes MDM set (three-state).", "why": "One canonical geography.", "loss": "Missing ZIP kept as NULL (unknown, not dropped)." }, { "op": "dedup", "detail": "ON CONFLICT(case_number, source_dataset) last-write-wins; source carries ~1.98 rows/case, collapsed to one case row.", "why": "Case grain — one row per foreclosure case, not per source line.", "loss": "Multi-line cases collapse to one (last-write-wins on the sale fields); case COUNTS are sound." } ], "produces_columns": [ "address", "zip_code", "neighborhood", "census_tract", "council_district", "parcel_id", "action_filed", "sale_date", "sale_price", "is_metro_purchase", "case_number", "source_dataset", "bronze_id" ], "reproduce": { "commands": [ "civic-data ingest-zips # canonical ZIP MDM first", "civic-data ingest foreclosures --promote # source → bronze → silver (single pull)" ], "gaps": [ "NO GREAT-RECESSION ERA — the source is dense only from ~2013; the 2008–2012 foreclosure wave is absent and not recoverable from this source. Trend use is honest only from ~2013 on.", "SOURCE GROWTH + CHURN — our pull (1,734 cases) SHARES 1,394 case numbers with civic-graph (1,493); +340 are newer/re-added and −99 have churned off since civic-graph's pull, while shared historical years match row-for-row (±1–2). 62 old-numbered cases (12-CI-…, 13-CI-…) carry NULL action_filed (57 new-to-source, 5 also in civic-graph) — a source condition, KEPT (not skipped), matching civic-graph's own treatment." ] }, "verified_against_schema": true, "note": null, "quarantine_live": 2 } }, { "key": "food_service_establishments", "title": "Permitted food service establishments", "trust_tier": "snapshot_only", "generated": { "schema": "silver", "table": "food_service_establishments", "row_count": 3366, "primary_key": [ "id" ], "year_column": null, "fields": [ { "name": "id", "type": "bigint", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "permit_number", "type": "integer", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "premise_name", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "premise_address", "type": "text", "nullable": true, "null_count": 222, "null_pct": 6.6 }, { "name": "city", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "zip_code", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "facility_type", "type": "integer", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "facility_type_description", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "subtype", "type": "integer", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "subtype_description", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "opening_date", "type": "text", "nullable": true, "null_count": 15, "null_pct": 0.4 }, { "name": "source_dataset", "type": "text", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "bronze_id", "type": "bigint", "nullable": true, "null_count": 0, "null_pct": 0.0 } ], "per_year": null, "enum_distributions": [ { "field": "facility_type_description", "distinct_values": 4, "null_count": 0, "null_pct": 0.0, "top_values": [ { "value": "FOOD SERVICE ESTABLISHMENT", "count": 2613, "pct": 77.6 }, { "value": "RETAIL/FOOD ESTABLISHMENT", "count": 357, "pct": 10.6 }, { "value": "RETAIL FOOD STORES", "count": 335, "pct": 10.0 }, { "value": "RESTRICTED FOOD CONCESSIONS", "count": 61, "pct": 1.8 } ], "tail_count": 0, "truncated": false }, { "field": "subtype_description", "distinct_values": 16, "null_count": 0, "null_pct": 0.0, "top_values": [ { "value": "FOOD SERVICE", "count": 1980, "pct": 58.8 }, { "value": "RETAIL-FOOD 10,000 SQ FT OR LESS", "count": 298, "pct": 8.9 }, { "value": "PRE-PACKAGED RETAIL", "count": 234, "pct": 7.0 }, { "value": "FOOD SERVICE (DAY CARE / CHILD CARE CENTER", "count": 181, "pct": 5.4 }, { "value": "SELF-CONTAINED MOBILE FOOD UNITS", "count": 167, "pct": 5.0 }, { "value": "SCHOOL CAFETERIA OR FOOD SERVICE", "count": 148, "pct": 4.4 }, { "value": "FOOD SERVICE (HOSPITAL / NURSING HOME", "count": 71, "pct": 2.1 }, { "value": "RETAIL-FOOD 10,001 SQ FT OR OVER", "count": 59, "pct": 1.8 }, { "value": "CATERERS", "count": 40, "pct": 1.2 }, { "value": "STATIONARY RESTRICTED CONCESSON", "count": 36, "pct": 1.1 }, { "value": "MOBILE RETAIL UNIT", "count": 32, "pct": 1.0 }, { "value": "SUPERMARKET WITH PROCESSING", "count": 30, "pct": 0.9 }, { "value": "RETAIL BAKERY", "count": 29, "pct": 0.9 }, { "value": "FOOD SERVICE COMMISSARY", "count": 26, "pct": 0.8 }, { "value": "MOBILE RESTRICTED CONCESSION", "count": 25, "pct": 0.7 }, { "value": "RETAIL WITH LIMITED SERVICE", "count": 10, "pct": 0.3 } ], "tail_count": 0, "truncated": false } ] }, "grain": "One row per permit. UNIQUE(permit_number, source_dataset).", "ingest_tool": "civic-data ingest food-service", "overlay": { "measures_vs_assumes": "Measures the CURRENT roster of permitted food-service establishments — who holds a permit now — not restaurant openings/closings over time. Carries opening_date but NO closure data, so it is an ambiguous time signal; treat it as a current-state roster.", "representativeness_bias": "Permitted establishments only; unpermitted/exempt food operations are invisible. Density tracks commercial food activity, weighted toward areas with formal businesses.", "pii_tier": "public-aggregate-safe", "license": "Louisville Metro Open Data — published under the Open Data Commons Public Domain Dedication and License (PDDL; verified 2026-07-11 at data.louisvilleky.gov/pages/terms-of-use-and-license, backed by Open Data Ordinance O-243-22). Share/create/adapt permitted, no attribution required; when republishing modified data, identify the source, version, and modifications made - which this project's generated lineage layer does per dataset. We license only our added value (normalization + documentation), never the city's underlying data - see LICENSE-data (CC BY 4.0).", "definitional_drift": "A living roster: establishments enter (new permits) and leave (closures) between pulls, so a fresh pull differs modestly from an earlier one (~96% overlap with civic-graph). opening_date is kept as the raw source string.", "as_of": "Single pull (current roster).", "cadence": "Source updated as permits are issued/closed; re-pulled on demand.", "provenance": "ArcGIS Feature Service — Permitted_Food_Service_Establishments (single pull). Premise names are business identifiers (kept)." }, "lineage": { "kind": "transactional", "source_system": "Louisville Metro / LOJIC ArcGIS Hub — 'Permitted_Food_Service_Establishments' Feature Service (single pull; a current permit roster).", "silver_table": "silver.food_service_establishments", "why_summary": "Turn the permitted food-service roster into one typed, ZIP-validated table — being explicit that it is a CURRENT roster of permit holders (opening_date but no closures), not a time series of openings/closings.", "transforms": [ { "op": "fetch", "detail": "Single ArcGIS query (~3.4K rows).", "why": "One pull captures the current roster.", "loss": "Fetch-stage gaps bounded by bronze; the roster itself churns (permits open/close)." }, { "op": "normalize_fields", "detail": "premise_name/address/city, facility_type(+description), subtype(+description) mapped to canonical columns; opening_date kept as raw text.", "why": "One typed silver schema.", "loss": "None — renaming/typing. opening_date not coerced to DATE (raw source string)." }, { "op": "validate_zip", "detail": "premise_zip validated against silver.zip_codes MDM (three-state).", "why": "One canonical geography.", "loss": "Missing ZIP kept NULL (0% here); non-Louisville quarantined." }, { "op": "dedup", "detail": "ON CONFLICT(permit_number, source_dataset) upsert.", "why": "One row per permit.", "loss": "None beyond de-duplication of re-pulled records." } ], "produces_columns": [ "permit_number", "premise_name", "premise_address", "city", "zip_code", "facility_type", "facility_type_description", "subtype", "subtype_description", "opening_date", "source_dataset", "bronze_id" ], "reproduce": { "commands": [ "civic-data ingest-zips # canonical ZIP MDM first", "civic-data ingest food-service --promote # source → bronze → silver (single pull)" ], "gaps": [ "CURRENT-ROSTER SOURCE — a fresh pull is who-holds-a-permit-now (3,366), ~96% overlapping civic-graph's earlier 3,391; the small difference is establishments entering/leaving between pulls. No closure data means no honest openings/closings trend from this source." ] }, "verified_against_schema": true, "note": null, "quarantine_live": 0 } }, { "key": "abc_licenses", "title": "ABC (alcohol) licenses — local + state", "trust_tier": "snapshot_only", "generated": { "schema": "silver", "table": "abc_licenses", "row_count": 7149, "primary_key": [ "id" ], "year_column": null, "fields": [ { "name": "id", "type": "bigint", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "license_number", "type": "text", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "business_name", "type": "text", "nullable": true, "null_count": 6, "null_pct": 0.1 }, { "name": "license_category", "type": "text", "nullable": true, "null_count": 5468, "null_pct": 76.5 }, { "name": "license_type", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "status", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "address", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "city", "type": "text", "nullable": true, "null_count": 1681, "null_pct": 23.5 }, { "name": "zip_code", "type": "text", "nullable": true, "null_count": 5, "null_pct": 0.1 }, { "name": "council_district", "type": "text", "nullable": true, "null_count": 5471, "null_pct": 76.5 }, { "name": "latitude", "type": "double precision", "nullable": true, "null_count": 7, "null_pct": 0.1 }, { "name": "longitude", "type": "double precision", "nullable": true, "null_count": 7, "null_pct": 0.1 }, { "name": "geom", "type": "USER-DEFINED", "nullable": true }, { "name": "issue_date", "type": "date", "nullable": true, "null_count": 1683, "null_pct": 23.5 }, { "name": "expiry_date", "type": "date", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "source_dataset", "type": "text", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "bronze_id", "type": "bigint", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "endorsements", "type": "text", "nullable": true, "null_count": 5556, "null_pct": 77.7 } ], "per_year": null, "enum_distributions": [ { "field": "license_type", "distinct_values": 50, "null_count": 0, "null_pct": 0.0, "top_values": [ { "value": "Alcoholic Beverage Annual License", "count": 1681, "pct": 23.5 }, { "value": "Special Sunday Retail Drink License", "count": 907, "pct": 12.7 }, { "value": "NQ Retail Malt Beverage Package License", "count": 849, "pct": 11.9 }, { "value": "Tobacco, Nicotine, or Vapor Product License", "count": 816, "pct": 11.4 }, { "value": "NQ2 Retail Drink License", "count": 585, "pct": 8.2 }, { "value": "NQ4 Retail Malt Beverage Drink License", "count": 517, "pct": 7.2 }, { "value": "Quota Retail Package License", "count": 423, "pct": 5.9 }, { "value": "Quota Retail Drink License", "count": 345, "pct": 4.8 }, { "value": "Sampling License", "count": 259, "pct": 3.6 }, { "value": "Supplemental Bar License", "count": 155, "pct": 2.2 }, { "value": "Caterer's License", "count": 75, "pct": 1.0 }, { "value": "Transporter's License", "count": 60, "pct": 0.8 }, { "value": "NQ3 Retail Drink License", "count": 59, "pct": 0.8 }, { "value": "Limited In-State Distilled Spirits Supplier's License", "count": 37, "pct": 0.5 }, { "value": "Distilled Spirits and Wine Storage License", "count": 36, "pct": 0.5 }, { "value": "NQ1 Retail Drink License", "count": 27, "pct": 0.4 }, { "value": "Microbrewery License", "count": 26, "pct": 0.4 }, { "value": "Vintage Distilled Spirits License", "count": 23, "pct": 0.3 }, { "value": "Direct Shipper License", "count": 22, "pct": 0.3 }, { "value": "Distiller's License - Class B", "count": 20, "pct": 0.3 }, { "value": "Bottling House or Bottling House Storage License", "count": 19, "pct": 0.3 }, { "value": "Extended Hours Supplemental License", "count": 18, "pct": 0.3 }, { "value": "Special Temporary Alcoholic Beverage Auction License", "count": 17, "pct": 0.2 }, { "value": "Limited Non Quota Package License", "count": 17, "pct": 0.2 }, { "value": "Wholesaler's License", "count": 15, "pct": 0.2 } ], "tail_count": 141, "truncated": true }, { "field": "status", "distinct_values": 4, "null_count": 0, "null_pct": 0.0, "top_values": [ { "value": "Active", "count": 7051, "pct": 98.6 }, { "value": "Out of Business", "count": 96, "pct": 1.3 }, { "value": "Closed", "count": 1, "pct": 0.0 }, { "value": "Revoked", "count": 1, "pct": 0.0 } ], "tail_count": 0, "truncated": false }, { "field": "license_category", "distinct_values": 22, "null_count": 5468, "null_pct": 76.5, "top_values": [ { "value": "Restaurant / Cafe", "count": 551, "pct": 7.7 }, { "value": "Grocery / Food Mart", "count": 352, "pct": 4.9 }, { "value": "Liquor Store", "count": 248, "pct": 3.5 }, { "value": "Bar", "count": 162, "pct": 2.3 }, { "value": "Retail", "count": 55, "pct": 0.8 }, { "value": "Hotels", "count": 54, "pct": 0.8 }, { "value": "Manufacturer", "count": 50, "pct": 0.7 }, { "value": "Private Club", "count": 42, "pct": 0.6 }, { "value": "Church / School", "count": 35, "pct": 0.5 }, { "value": "Drug Store", "count": 26, "pct": 0.4 }, { "value": "Catering", "count": 17, "pct": 0.2 }, { "value": "Amusement", "count": 16, "pct": 0.2 }, { "value": "Golf Course", "count": 14, "pct": 0.2 }, { "value": "Theatre / Museum", "count": 12, "pct": 0.2 }, { "value": "Distributor / Wholesaler", "count": 10, "pct": 0.1 }, { "value": "Country Club", "count": 10, "pct": 0.1 }, { "value": "Convention Center Complex", "count": 9, "pct": 0.1 }, { "value": "Airport", "count": 6, "pct": 0.1 }, { "value": "River Boat", "count": 4, "pct": 0.1 }, { "value": "Sports / Fitness", "count": 4, "pct": 0.1 }, { "value": "Bowling Alley", "count": 3, "pct": 0.0 }, { "value": "Horse Race Track", "count": 1, "pct": 0.0 } ], "tail_count": 0, "truncated": false } ] }, "grain": "One row per license. UNIQUE(license_number, source_dataset). TWO sources: abc-local-licenses (Metro) + abc-state-licenses.", "ingest_tool": "civic-data ingest abc", "overlay": { "measures_vs_assumes": "Measures CURRENTLY-active alcohol licenses (a commercial nightlife/dining signal) from two authorities — Metro (local) + state — not a time series of licensing activity. A lapsed/expired license leaves the active set.", "representativeness_bias": "Two overlapping authorities with different fields; the local subset carries endorsement detail, the state subset does not. Reading license density as 'nightlife intensity' conflates permit count with actual activity.", "pii_tier": "public-aggregate-safe", "license": "Louisville Metro Open Data — published under the Open Data Commons Public Domain Dedication and License (PDDL; verified 2026-07-11 at data.louisvilleky.gov/pages/terms-of-use-and-license, backed by Open Data Ordinance O-243-22). Share/create/adapt permitted, no attribution required; when republishing modified data, identify the source, version, and modifications made - which this project's generated lineage layer does per dataset. We license only our added value (normalization + documentation), never the city's underlying data - see LICENSE-data (CC BY 4.0).", "definitional_drift": "Active-license lists CHURN (renewals/expiries) + grow modestly between pulls. ENRICHMENT: the endorsements dataset fills license_category (only when NULL — ingest wins) and a semicolon-joined endorsement list on the LOCAL subset only; state licenses have no endorsement source and keep pattern-based classification (this is a CLASSIFICATION-coverage lift, not a universe lift). ZIP is parsed from a free address string (local) or PremisesCityState (state).", "as_of": "Single pull each source (current active licenses).", "cadence": "Continuously updated at source; re-pulled on demand.", "provenance": "ArcGIS Feature Services — abc_license_ait4135 (local) + ABC_State_ActiveLicenses (state) + abc_license_endorsements (enrichment). Business/DBA names are business identifiers (kept)." }, "lineage": { "kind": "transactional", "source_system": "Louisville Metro / LOJIC ArcGIS Hub — 'abc_license_ait4135' (local) + 'ABC_State_ActiveLicenses' (state) Feature Services, enriched from 'abc_license_endorsements'. Single pull each.", "silver_table": "silver.abc_licenses", "why_summary": "Merge two active-license authorities (Metro + state) into one typed, geocoded, ZIP-validated table and enrich the local subset with endorsement classification — being explicit that it is a CURRENT active-license footprint, not a licensing time series.", "transforms": [ { "op": "fetch", "detail": "One ArcGIS query per source (local ~1.7K, state ~5.5K).", "why": "One pull per authority captures the current active set.", "loss": "Active-license lists churn (renewals/expiries); a lapsed license leaves the set." }, { "op": "normalize_fields", "detail": "Two different source schemas mapped to one: local (LicenseNO/BusinessName/LicenseCat/LicStatus/GPSX-Y/EXPDTTM/CouncilDist) and state (LicenseNumber/DBA/Status/PremisesStreet/City/Latitude-Longitude/IssueDate/ExpiryDate).", "why": "One silver schema across two authorities.", "loss": "Fields absent in one source are NULL there (local has no city/issue_date; state has no license_category/council_district)." }, { "op": "derive", "detail": "ZIP parsed as the last 5-digit token of the free address string (local) or PremisesCityState (state).", "why": "Neither source ships a clean ZIP field.", "loss": "Unparseable → left for validate_zip to NULL." }, { "op": "build_geom", "detail": "ST_MakePoint(lng, lat) → geom from the source coordinates.", "why": "Both sources are geocoded — a point enables sub-ZIP placement.", "loss": "NULL geom where coordinates absent." }, { "op": "validate_zip", "detail": "ZIP validated against silver.zip_codes MDM (three-state).", "why": "One canonical geography.", "loss": "Missing kept NULL (~0.1%); non-Louisville quarantined (mailing vs premises ZIP)." }, { "op": "dedup", "detail": "ON CONFLICT(license_number, source_dataset) upsert.", "why": "One row per license per authority.", "loss": "None beyond de-duplication of re-pulled records." }, { "op": "derive", "detail": "ENRICH (post-promote): the endorsements dataset fills license_category (only when NULL — ingest wins) and a semicolon-joined endorsement list on the LOCAL subset.", "why": "Improves restaurant-vs-retail CLASSIFICATION accuracy on the local subset.", "loss": "Classification-coverage lift only — NOT a universe lift; state licenses have no endorsement source and keep pattern-based classification." } ], "produces_columns": [ "license_number", "business_name", "license_category", "license_type", "status", "address", "city", "zip_code", "council_district", "latitude", "longitude", "geom", "issue_date", "expiry_date", "source_dataset", "bronze_id", "endorsements" ], "reproduce": { "commands": [ "civic-data ingest-zips # canonical ZIP MDM first", "civic-data ingest abc --promote # local + state → bronze → silver, then endorsement enrich" ], "gaps": [ "ACTIVE-LICENSE SOURCE — a fresh pull is the current active set (7,149: local 1,681 / state 5,468), high-overlap with civic-graph's earlier 7,038 (local 1,635 / state 5,403), differing by renewal/expiry churn + modest growth. Endorsement enrichment reproduced (1,593 local licenses enriched vs civic-graph's 1,556). Not a licensing time series." ] }, "verified_against_schema": true, "note": null, "quarantine_live": 15 } }, { "key": "restaurant_inspections", "title": "Restaurant inspection scores", "trust_tier": "viewable_caveated", "generated": { "schema": "silver", "table": "restaurant_inspections", "row_count": 11273, "primary_key": [ "id" ], "year_column": "inspection_date", "fields": [ { "name": "id", "type": "bigint", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "establishment_id", "type": "integer", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "inspection_id", "type": "integer", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "inspection_type", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "establishment_name", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "address", "type": "text", "nullable": true, "null_count": 1964, "null_pct": 17.4 }, { "name": "city", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "zip_code", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "establishment_type", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "inspection_date", "type": "date", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "score", "type": "integer", "nullable": true, "null_count": 586, "null_pct": 5.2 }, { "name": "grade", "type": "text", "nullable": true, "null_count": 4095, "null_pct": 36.3 }, { "name": "source_dataset", "type": "text", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "bronze_id", "type": "bigint", "nullable": true, "null_count": 0, "null_pct": 0.0 } ], "per_year": [ { "year": 2025, "count": 6004 }, { "year": 2026, "count": 5269 } ], "enum_distributions": [ { "field": "grade", "distinct_values": 3, "null_count": 4095, "null_pct": 36.3, "top_values": [ { "value": "A", "count": 6790, "pct": 60.2 }, { "value": "C", "count": 351, "pct": 3.1 }, { "value": "B", "count": 37, "pct": 0.3 } ], "tail_count": 0, "truncated": false }, { "field": "establishment_type", "distinct_values": 19, "null_count": 0, "null_pct": 0.0, "top_values": [ { "value": "FOOD SERVICE", "count": 5452, "pct": 48.4 }, { "value": "RETAIL-FOOD 10,000 SQ FT OR LESS", "count": 1596, "pct": 14.2 }, { "value": "TEMPORARY FOODS", "count": 1406, "pct": 12.5 }, { "value": "FOOD SERVICE (DAY CARE / CHILD CARE CENTER", "count": 459, "pct": 4.1 }, { "value": "SELF-CONTAINED MOBILE FOOD UNITS", "count": 453, "pct": 4.0 }, { "value": "PRE-PACKAGED RETAIL", "count": 444, "pct": 3.9 }, { "value": "SCHOOL CAFETERIA OR FOOD SERVICE", "count": 421, "pct": 3.7 }, { "value": "RETAIL-FOOD 10,001 SQ FT OR OVER", "count": 367, "pct": 3.3 }, { "value": "FOOD SERVICE (HOSPITAL / NURSING HOME", "count": 168, "pct": 1.5 }, { "value": "CATERERS", "count": 93, "pct": 0.8 }, { "value": "FOOD SERVICE COMMISSARY", "count": 88, "pct": 0.8 }, { "value": "SUPERMARKET WITH PROCESSING", "count": 78, "pct": 0.7 }, { "value": "STATIONARY RESTRICTED CONCESSON", "count": 77, "pct": 0.7 }, { "value": "MOBILE RETAIL UNIT", "count": 66, "pct": 0.6 }, { "value": "MOBILE RESTRICTED CONCESSION", "count": 41, "pct": 0.4 }, { "value": "RETAIL BAKERY", "count": 34, "pct": 0.3 }, { "value": "RETAIL WITH LIMITED SERVICE", "count": 15, "pct": 0.1 }, { "value": "FARMERS MARKET TEMPORARY", "count": 14, "pct": 0.1 }, { "value": "VENDING MACHINE COMPANY", "count": 1, "pct": 0.0 } ], "tail_count": 0, "truncated": false } ] }, "grain": "One row per inspection. UNIQUE(inspection_id, source_dataset); an establishment recurs across inspections.", "ingest_tool": "civic-data ingest inspections", "overlay": { "measures_vs_assumes": "Measures health-inspection events + scores within the source's window — not overall food safety nor a full inspection history. A score is one inspection at one time.", "representativeness_bias": "Inspection frequency is regulator-driven; more-inspected categories appear more. Score distributions reflect enforcement cadence, not just establishment quality.", "pii_tier": "public-aggregate-safe", "license": "Louisville Metro Open Data — published under the Open Data Commons Public Domain Dedication and License (PDDL; verified 2026-07-11 at data.louisvilleky.gov/pages/terms-of-use-and-license, backed by Open Data Ordinance O-243-22). Share/create/adapt permitted, no attribution required; when republishing modified data, identify the source, version, and modifications made - which this project's generated lineage layer does per dataset. We license only our added value (normalization + documentation), never the city's underlying data - see LICENSE-data (CC BY 4.0).", "definitional_drift": "ROLLING ~12-MONTH WINDOW: the source holds only about the last year of inspections (our pull 2025-07→2026-07; civic-graph's 2025-03→2026-03). A single pull is a recent window, NOT a multi-year archive — do not build a long time series from one pull. The window shift between pulls explains partial overlap with civic-graph (both hold ~11K, sharing ~7.9K).", "as_of": "Single pull (rolling ~12-month window).", "cadence": "Continuously updated; oldest inspections roll off the window.", "provenance": "ArcGIS Feature Service — FoodServiceData (single pull). Establishment names are business identifiers (kept)." }, "lineage": { "kind": "transactional", "source_system": "Louisville Metro / LOJIC ArcGIS Hub — 'FoodServiceData' Feature Service (single pull; a rolling ~12-month window).", "silver_table": "silver.restaurant_inspections", "why_summary": "Turn the inspection-scores feed into one typed, ZIP-validated table — being explicit that the source holds only a ROLLING ~12-month window, so a single pull is a recent window, not a multi-year inspection history.", "transforms": [ { "op": "fetch", "detail": "Single ArcGIS query (~11K rows).", "why": "One pull captures the current ~12-month window.", "loss": "ROLLING WINDOW, not fetch loss: inspections older than ~12 months have already rolled off." }, { "op": "normalize_fields", "detail": "EstablishmentID/InspectionID, Ins_TypeDesc, EstablishmentName, TypeDescription, score, Grade mapped to canonical columns.", "why": "One typed silver schema.", "loss": "Records with an empty establishment name are SKIPPED." }, { "op": "parse", "detail": "InspectionDate '2025-06-04 00:00:00' truncated to the leading DATE.", "why": "A typed inspection_date is the temporal grain.", "loss": "None — deterministic truncation." }, { "op": "validate_zip", "detail": "Zip zero-padded to 5, validated against silver.zip_codes MDM (three-state).", "why": "One canonical geography.", "loss": "Missing kept NULL (~0%); non-Louisville quarantined." }, { "op": "dedup", "detail": "ON CONFLICT(inspection_id, source_dataset) upsert.", "why": "One row per inspection.", "loss": "None beyond de-duplication of re-pulled records." } ], "produces_columns": [ "establishment_id", "inspection_id", "inspection_type", "establishment_name", "address", "city", "zip_code", "establishment_type", "inspection_date", "score", "grade", "source_dataset", "bronze_id" ], "reproduce": { "commands": [ "civic-data ingest-zips # canonical ZIP MDM first", "civic-data ingest inspections --promote # source → bronze → silver (rolling window)" ], "gaps": [ "ROLLING ~12-MONTH WINDOW — the source holds only the last ~year of inspections (our pull 2025-07→2026-07; civic-graph's 2025-03→2026-03). A single pull cannot reproduce a multi-year archive; the ~4-month window shift explains the partial overlap (both ~11K, sharing ~7.9K). A longitudinal inspection record requires accumulating windows over time." ] }, "verified_against_schema": true, "note": null, "quarantine_live": 0 } }, { "key": "construction_permits", "title": "Active construction permits", "trust_tier": "viewable_caveated", "generated": { "schema": "silver", "table": "construction_permits", "row_count": 23308, "primary_key": [ "id" ], "year_column": "issue_date", "fields": [ { "name": "id", "type": "bigint", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "permit_number", "type": "text", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "permit_type", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "permit_status", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "category", "type": "text", "nullable": true, "null_count": 14012, "null_pct": 60.1 }, { "name": "work_type", "type": "text", "nullable": true, "null_count": 12349, "null_pct": 53.0 }, { "name": "zoning", "type": "text", "nullable": true, "null_count": 137, "null_pct": 0.6 }, { "name": "sqft", "type": "double precision", "nullable": true, "null_count": 14736, "null_pct": 63.2 }, { "name": "permit_fee", "type": "double precision", "nullable": true, "null_count": 66, "null_pct": 0.3 }, { "name": "project_costs", "type": "double precision", "nullable": true, "null_count": 2970, "null_pct": 12.7 }, { "name": "address", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "zip_code", "type": "text", "nullable": true, "null_count": 37, "null_pct": 0.2 }, { "name": "latitude", "type": "double precision", "nullable": true, "null_count": 62, "null_pct": 0.3 }, { "name": "longitude", "type": "double precision", "nullable": true, "null_count": 62, "null_pct": 0.3 }, { "name": "geom", "type": "USER-DEFINED", "nullable": true }, { "name": "council_district", "type": "text", "nullable": true, "null_count": 120, "null_pct": 0.5 }, { "name": "neighborhood", "type": "text", "nullable": true, "null_count": 20321, "null_pct": 87.2 }, { "name": "issue_date", "type": "date", "nullable": true, "null_count": 54, "null_pct": 0.2 }, { "name": "source_dataset", "type": "text", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "bronze_id", "type": "bigint", "nullable": true, "null_count": 0, "null_pct": 0.0 } ], "per_year": [ { "year": 2014, "count": 1099 }, { "year": 2015, "count": 1032 }, { "year": 2016, "count": 1297 }, { "year": 2017, "count": 1318 }, { "year": 2018, "count": 1658 }, { "year": 2019, "count": 2599 }, { "year": 2020, "count": 232 }, { "year": 2021, "count": 54 }, { "year": 2022, "count": 157 }, { "year": 2023, "count": 280 }, { "year": 2024, "count": 3145 }, { "year": 2025, "count": 4711 }, { "year": 2026, "count": 5672 } ], "enum_distributions": [ { "field": "permit_type", "distinct_values": 29, "null_count": 0, "null_pct": 0.0, "top_values": [ { "value": "HVAC Residential", "count": 5638, "pct": 24.2 }, { "value": "Electrical Residential", "count": 4259, "pct": 18.3 }, { "value": "Residential Alteration", "count": 2435, "pct": 10.4 }, { "value": "Residential New", "count": 1727, "pct": 7.4 }, { "value": "Residential Addition", "count": 1718, "pct": 7.4 }, { "value": "Electrical Commercial", "count": 1594, "pct": 6.8 }, { "value": "HVAC Commercial", "count": 1255, "pct": 5.4 }, { "value": "Commercial Alteration", "count": 1130, "pct": 4.8 }, { "value": "PoolSpa Residential", "count": 488, "pct": 2.1 }, { "value": "Fire Suppression System", "count": 411, "pct": 1.8 }, { "value": "Commercial New", "count": 402, "pct": 1.7 }, { "value": "Change of Use", "count": 347, "pct": 1.5 }, { "value": "Commercial Addition", "count": 288, "pct": 1.2 }, { "value": "Wrecking Permit", "count": 277, "pct": 1.2 }, { "value": "Fire Detection", "count": 252, "pct": 1.1 }, { "value": "Fireplace", "count": 215, "pct": 0.9 }, { "value": "Tent", "count": 211, "pct": 0.9 }, { "value": "Residential Foundation", "count": 141, "pct": 0.6 }, { "value": "Commercial Foundation", "count": 127, "pct": 0.5 }, { "value": "Parking Lot", "count": 122, "pct": 0.5 }, { "value": "Cell Tower", "count": 85, "pct": 0.4 }, { "value": "Mechanical Refrigeration", "count": 81, "pct": 0.3 }, { "value": "Range Hood", "count": 27, "pct": 0.1 }, { "value": "Gas Restoration Pemit", "count": 27, "pct": 0.1 }, { "value": "PoolSpa Commercial", "count": 17, "pct": 0.1 } ], "tail_count": 34, "truncated": true }, { "field": "permit_status", "distinct_values": 1, "null_count": 0, "null_pct": 0.0, "top_values": [ { "value": "Issued", "count": 23308, "pct": 100.0 } ], "tail_count": 0, "truncated": false }, { "field": "category", "distinct_values": 6, "null_count": 14012, "null_pct": 60.1, "top_values": [ { "value": "Single Family", "count": 7734, "pct": 33.2 }, { "value": "Commercial", "count": 888, "pct": 3.8 }, { "value": "Multi Family", "count": 301, "pct": 1.3 }, { "value": "Multifamily", "count": 250, "pct": 1.1 }, { "value": "Condo/Patio", "count": 109, "pct": 0.5 }, { "value": "Multi-Family", "count": 14, "pct": 0.1 } ], "tail_count": 0, "truncated": false } ] }, "grain": "One row per permit. UNIQUE(permit_number, source_dataset). Geocoded at source.", "ingest_tool": "civic-data ingest permits", "overlay": { "measures_vs_assumes": "Measures ACTIVE construction permits (an investment/development signal), not all permits ever issued nor completed construction. A permit that closes leaves the active set.", "representativeness_bias": "Permit activity concentrates where formal development happens; informal or unpermitted work is invisible. Project_costs are self-declared at application.", "pii_tier": "public-aggregate-safe", "license": "Louisville Metro Open Data — published under the Open Data Commons Public Domain Dedication and License (PDDL; verified 2026-07-11 at data.louisvilleky.gov/pages/terms-of-use-and-license, backed by Open Data Ordinance O-243-22). Share/create/adapt permitted, no attribution required; when republishing modified data, identify the source, version, and modifications made - which this project's generated lineage layer does per dataset. We license only our added value (normalization + documentation), never the city's underlying data - see LICENSE-data (CC BY 4.0).", "definitional_drift": "ACTIVE-permit rolling set: historical issue-years are stable (2019 matches civic-graph within ~5), but recent composition shifts as permits close and leave while new ones enter (between civic-graph's pull and ours, 2025 dropped ~3,063 and 2026 gained ~2,751). PLUS a known 2020–2022 record-capture discontinuity (the source under-publishes those years) — do NOT read 2019→2020 permit movement as a real economic signal. contractor names stripped at ingest.", "as_of": "Single pull (active set).", "cadence": "Continuously updated; closed permits leave the active set.", "provenance": "ArcGIS Feature Service — active_construction_permits (single pull). Contractor (possible sole-proprietor name) stripped before any DB write." }, "lineage": { "kind": "transactional", "source_system": "Louisville Metro / LOJIC ArcGIS Hub — 'active_construction_permits' Feature Service (single pull; a live ACTIVE-permit set).", "silver_table": "silver.construction_permits", "why_summary": "Turn the active-construction-permits feed into one typed, geocoded, ZIP-validated table with project cost + neighborhood — being explicit that it is the ACTIVE set (closed permits leave) and that 2020–2022 has a source capture discontinuity.", "transforms": [ { "op": "fetch", "detail": "Single ArcGIS query (~23K rows).", "why": "One pull captures the current active-permit set.", "loss": "ACTIVE SET, not fetch loss: permits that have closed have already left the source." }, { "op": "strip_pii", "detail": "CONTRACTOR (possible sole-proprietor name) dropped before any DB write.", "why": "No-PII-in-DB policy.", "loss": "None relevant to analysis; the site address is retained." }, { "op": "normalize_fields", "detail": "PERMIT_TYPE/STATUS, CATEGORY_NAME, WORK_TYPE, ZONING, SQFT, PERMIT_FEE, PROJECT_COSTS, DISTRICT, NEIGHBORHOOD mapped to canonical columns.", "why": "One typed silver schema.", "loss": "None — renaming/typing. project_costs are self-declared at application." }, { "op": "parse", "detail": "ISSUE_DATE epoch-ms → DATE.", "why": "A typed issue_date is the temporal grain.", "loss": "None." }, { "op": "build_geom", "detail": "ST_MakePoint(lng, lat) → geom from the source coordinates.", "why": "Permits are geocoded at source — a point enables sub-ZIP placement and district joins.", "loss": "NULL geom where coordinates absent." }, { "op": "validate_zip", "detail": "ZIPCODE validated against silver.zip_codes MDM (three-state).", "why": "One canonical geography.", "loss": "Missing kept NULL (~0.2%); non-Louisville quarantined." }, { "op": "dedup", "detail": "ON CONFLICT(permit_number, source_dataset) upsert.", "why": "One row per permit.", "loss": "None beyond de-duplication of re-pulled records." } ], "produces_columns": [ "permit_number", "permit_type", "permit_status", "category", "work_type", "zoning", "sqft", "permit_fee", "project_costs", "address", "zip_code", "latitude", "longitude", "geom", "council_district", "neighborhood", "issue_date", "source_dataset", "bronze_id" ], "reproduce": { "commands": [ "civic-data ingest-zips # canonical ZIP MDM first", "civic-data ingest permits --promote # source → bronze → silver (active set)" ], "gaps": [ "ACTIVE-PERMIT ROLLING SET — historical issue-years are stable (2019 matches civic-graph within ~5), but recent composition shifts as permits close and leave while new ones enter (between civic-graph's pull and ours, 2025 dropped ~3,063 and 2026 gained ~2,751); totals stay near ~24K. Overlap with civic-graph is ~19.2K of ~23–24K each.", "2020–2022 CAPTURE DISCONTINUITY — the source under-publishes those years (a records artifact, not an economic signal). Do NOT read 2019→2020 permit movement as real." ] }, "verified_against_schema": true, "note": null, "quarantine_live": 8 } }, { "key": "str_licenses", "title": "Short-Term Rental registrations", "trust_tier": "snapshot_only", "generated": { "schema": "silver", "table": "str_licenses", "row_count": 1248, "primary_key": [ "id" ], "year_column": "date_opened", "fields": [ { "name": "id", "type": "bigint", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "license_number", "type": "text", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "address", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "zip_code", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "council_district", "type": "integer", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "police_district", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "zoning_code", "type": "text", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "date_opened", "type": "date", "nullable": true, "null_count": 0, "null_pct": 0.0 }, { "name": "latitude", "type": "double precision", "nullable": true, "null_count": 1, "null_pct": 0.1 }, { "name": "longitude", "type": "double precision", "nullable": true, "null_count": 1, "null_pct": 0.1 }, { "name": "geom", "type": "USER-DEFINED", "nullable": true }, { "name": "source_dataset", "type": "text", "nullable": false, "null_count": 0, "null_pct": 0.0 }, { "name": "bronze_id", "type": "bigint", "nullable": true, "null_count": 0, "null_pct": 0.0 } ], "per_year": [ { "year": 2025, "count": 540 }, { "year": 2026, "count": 708 } ], "enum_distributions": [ { "field": "zoning_code", "distinct_values": 28, "null_count": 0, "null_pct": 0.0, "top_values": [ { "value": "R5", "count": 307, "pct": 24.6 }, { "value": "R6", "count": 134, "pct": 10.7 }, { "value": "R4", "count": 101, "pct": 8.1 }, { "value": "OR2", "count": 95, "pct": 7.6 }, { "value": "C1", "count": 92, "pct": 7.4 }, { "value": "(C3)", "count": 83, "pct": 6.7 }, { "value": "(C2)", "count": 72, "pct": 5.8 }, { "value": "TNZD", "count": 57, "pct": 4.6 }, { "value": "C2", "count": 53, "pct": 4.2 }, { "value": "EZ1", "count": 48, "pct": 3.8 }, { "value": "R5B", "count": 41, "pct": 3.3 }, { "value": "R5A", "count": 36, "pct": 2.9 }, { "value": "R7", "count": 27, "pct": 2.2 }, { "value": "UN", "count": 22, "pct": 1.8 }, { "value": "OR3", "count": 19, "pct": 1.5 }, { "value": "PVD", "count": 18, "pct": 1.4 }, { "value": "CM", "count": 7, "pct": 0.6 }, { "value": "C3", "count": 6, "pct": 0.5 }, { "value": "(OR1)", "count": 6, "pct": 0.5 }, { "value": "(R4)", "count": 5, "pct": 0.4 }, { "value": "R3", "count": 5, "pct": 0.4 }, { "value": "CR", "count": 4, "pct": 0.3 }, { "value": "CN", "count": 3, "pct": 0.2 }, { "value": "R2", "count": 2, "pct": 0.2 }, { "value": "OR1", "count": 2, "pct": 0.2 } ], "tail_count": 3, "truncated": true }, { "field": "police_district", "distinct_values": 12, "null_count": 0, "null_pct": 0.0, "top_values": [ { "value": "5", "count": 392, "pct": 31.4 }, { "value": "4", "count": 348, "pct": 27.9 }, { "value": "1", "count": 260, "pct": 20.8 }, { "value": "6", "count": 100, "pct": 8.0 }, { "value": "8", "count": 62, "pct": 5.0 }, { "value": "7", "count": 34, "pct": 2.7 }, { "value": "3", "count": 30, "pct": 2.4 }, { "value": "ST MATTHEWS", "count": 8, "pct": 0.6 }, { "value": "2", "count": 6, "pct": 0.5 }, { "value": "AUDUBON PARK", "count": 4, "pct": 0.3 }, { "value": "MIDDLETOWN", "count": 3, "pct": 0.2 }, { "value": "LYNDON", "count": 1, "pct": 0.1 } ], "tail_count": 0, "truncated": false } ] }, "grain": "One row per registered STR license. UNIQUE(license_number, source_dataset).", "ingest_tool": "civic-data ingest str", "overlay": { "measures_vs_assumes": "Measures CURRENTLY-registered short-term rentals (a supply-side housing- pressure signal — residential addresses pulled toward the STR market), not the full STR footprint. Unregistered/unpermitted STRs are invisible, so this understates true STR pressure.", "representativeness_bias": "Registration compliance varies; areas with more enforcement/compliance over-report. Reading registered-STR density as total STR pressure understates the unlicensed gap.", "pii_tier": "person-naming", "license": "Louisville Metro Open Data — published under the Open Data Commons Public Domain Dedication and License (PDDL; verified 2026-07-11 at data.louisvilleky.gov/pages/terms-of-use-and-license, backed by Open Data Ordinance O-243-22). Share/create/adapt permitted, no attribution required; when republishing modified data, identify the source, version, and modifications made - which this project's generated lineage layer does per dataset. We license only our added value (normalization + documentation), never the city's underlying data - see LICENSE-data (CC BY 4.0).", "definitional_drift": "SOURCE CARRIES PII — host name/email/phone + owner name — ALL stripped at ingest (never persist in bronze, silver, or quarantine). A living registration roster: licenses enter (new registrations) and leave (closures) between pulls, so a fresh pull is a current-state snapshot. Address is block-level (ADDR_FULL_BLOCK, no unit). ZIP is derived spatially from the point (point-in-polygon), not the source. This is a DIFFERENT, better source than civic-graph's token-gated conditional-use-permit dataset (which it supersedes).", "as_of": "Single public pull (current registrations; source updated continuously).", "cadence": "Continuously updated at source; re-pulled on demand (public, no token).", "provenance": "ArcGIS Feature Service — short_term_rental_registration (single pull, PUBLIC). HOSTNAME/HOSTEMAIL/HOSTPHONE/OWNERNAME stripped before any DB write; block address + license kept." }, "lineage": { "kind": "transactional", "source_system": "Louisville Metro / LOJIC ArcGIS Hub — 'short_term_rental_registration' Feature Service (single PUBLIC pull; supersedes civic-graph's token-gated conditional-use dataset).", "silver_table": "silver.str_licenses", "why_summary": "Turn the registered-STR feed into one typed, spatially-ZIP'd table as a supply-side housing-pressure signal — while STRIPPING all four private-individual fields the source carries (host name/email/phone + owner name) so no PII ever persists, and being explicit that it is a current registration roster (not the full unlicensed STR footprint).", "transforms": [ { "op": "fetch", "detail": "Single ArcGIS query of the PUBLIC registration service (no token; ~1.25K).", "why": "Public + live means genuine from-source reproduction — a real upgrade over civic-graph's token-gated conditional-use archive, which this dataset supersedes.", "loss": "Registration roster churns (licenses open/close); a fresh pull is a current snapshot." }, { "op": "strip_pii", "detail": "HOSTNAME, HOSTEMAIL, HOSTPHONE, OWNERNAME dropped BEFORE any DB write (validation.strip_pii_keys) — never reach bronze, silver, or quarantine.", "why": "Hard no-PII-in-DB policy: private hosts' names, emails, and phone numbers never persist, even though the city publishes them openly.", "loss": "Four private fields intentionally not retained; the block address + license number are kept (a permit is a public act at a public address)." }, { "op": "normalize_fields", "detail": "LICENSE_NUMBER (key), ADDR_FULL_BLOCK → one-line block address, COUNCILDISTRICT → int, POLICEDISTRICT, ZONINGCODE, DATEOPENED epoch-ms → DATE.", "why": "One typed silver schema fit to this registration source.", "loss": "None — assembling/typing. ADDR_FULL_BLOCK is block-level (no unit)." }, { "op": "geocode", "detail": "ZIP DERIVED spatially: derive_zip_from_point(lng, lat) point-in-polygon against silver.zip_codes (the source ZIP is not trusted).", "why": "A geographic ZIP that came FROM the canonical set, not the source's own field.", "loss": "A point outside the 41-ZIP set → NULL zip (kept, not quarantined) — ~1 record." }, { "op": "build_geom", "detail": "ST_MakePoint(lng, lat) → geom from the source coordinates.", "why": "Point geometry enables sub-ZIP placement and district joins.", "loss": "NULL geom only where coordinates absent." }, { "op": "dedup", "detail": "ON CONFLICT(license_number, source_dataset) upsert.", "why": "One row per registered license.", "loss": "Duplicate license rows in the pull collapse to one (~4)." } ], "produces_columns": [ "license_number", "address", "zip_code", "council_district", "police_district", "zoning_code", "date_opened", "latitude", "longitude", "geom", "source_dataset", "bronze_id" ], "reproduce": { "commands": [ "civic-data ingest-zips # canonical ZIP MDM first (spatial ZIP derivation needs it)", "civic-data ingest str --promote # PUBLIC source → bronze (PII-stripped) → silver" ], "gaps": [ "DIFFERENT + BETTER SOURCE than civic-graph. civic-graph used the token-gated conditional-use-permit dataset via a captured 2026-05 GeoJSON (1,089 rows, PII-free). civic-data pulls the PUBLIC, live registration service (1,252 rows, PII-laden → stripped). The two are not row-comparable — different datasets. This one is fresher and genuinely re-pullable from source.", "CURRENT-ROSTER SOURCE — registered STRs only; the large unlicensed/unregistered footprint is invisible, so density understates true STR pressure. A single pull is a current snapshot, not a time series." ] }, "verified_against_schema": true, "note": null, "quarantine_live": 0 } } ], "reference_tables": [ { "schema": "silver", "table": "zip_codes", "row_count": 41, "note": "Canonical 41-ZIP Louisville boundary set (ArcGIS portal). MASTER/REFERENCE data (MDM): every event table validates its ZIP against this set. See the lineage layer's MDM record." }, { "schema": "silver", "table": "address_points", "row_count": 450308, "note": "LOJIC Jefferson County address points (~450K). Geocoding BUILD-AID (MDM-lite): resolves LMPD crime block-addresses (no coordinates at source) to a point via street_key + hundred-block range. Full-refresh from the LOJIC MapServer; not a consumer domain." }, { "schema": "silver", "table": "street_intersections", "row_count": 21048, "note": "LOJIC Jefferson County street intersections (~21K). Geocoding BUILD-AID: resolves crime intersection-addresses ('A ST / B AVE') to the intersection point via a sorted normalized street-key pair. Full-refresh from the LOJIC MapServer." }, { "schema": "silver", "table": "zip_to_district", "row_count": 133, "note": "DERIVED crosswalk: ZIP ↔ Council district area overlap (bidirectional coverage fractions, ::geography, sliver-filtered >0.001). Built by `ingest-geo` from the boundary sets; version-scoped. Lets ZIP-grained events roll up to districts. Matches civic-graph exactly (133)." }, { "schema": "silver", "table": "zip_to_neighborhood", "row_count": 318, "note": "DERIVED crosswalk: ZIP ↔ neighborhood area overlap. Built by `ingest-geo` (PostGIS overlay). Matches civic-graph exactly (318)." }, { "schema": "silver", "table": "zip_to_hud_zone", "row_count": 159, "note": "DERIVED crosswalk: ZIP ↔ HUD 1979 grant-zone area overlap. Built by `ingest-geo` (PostGIS overlay); urban-core coverage only. Matches civic-graph exactly (159)." } ] }