From 110cb11c464b4b2c97129d4724af9f5212832552 Mon Sep 17 00:00:00 2001 From: Rajendra Adhikari Date: Mon, 9 Nov 2026 13:34:54 -0600 Subject: [PATCH 01/13] Add data center one-shot extraction plugin Adds the data center schema and plugin config, post-processing for data center type labels, and scripts that export the human-extracted reference ordinances. Also loads jurisdictions by FIPS when the input file has a FIPS column, keeps leading zeros in known-source lookups, and stops definitions-only documents from counting as an ordinance. --- collapse_human_dc_ordinances.py | 409 ++++++++++++++++++ compass/__init__.py | 1 + compass/extraction/__init__.py | 1 + compass/extraction/data_centers/__init__.py | 12 + .../data_centers/data_centers_schema.json | 325 ++++++++++++++ .../data_centers/plugin_config.yaml | 164 +++++++ compass/pipeline/coordinator.py | 20 +- compass/pipeline/runtime.py | 2 +- compass/plugin/one_shot/components.py | 11 +- compass/plugin/ordinance.py | 2 + compass/plugin/post_processing.py | 35 ++ compass/utilities/costs.py | 1 + compass/utilities/jurisdictions.py | 39 +- export_human_dc_ordinances.py | 212 +++++++++ ...test_integrated_utilities_jurisdictions.py | 23 + .../test_extraction_data_centers_schema.py | 79 ++++ .../pipeline/test_pipeline_orchestration.py | 61 +++ .../unit/plugin/test_plugin_ordinances.py | 15 + .../plugin/test_plugin_post_processing.py | 99 +++++ .../utilities/test_utilities_jurisdictions.py | 97 +++++ 20 files changed, 1595 insertions(+), 13 deletions(-) create mode 100644 collapse_human_dc_ordinances.py create mode 100644 compass/extraction/data_centers/__init__.py create mode 100644 compass/extraction/data_centers/data_centers_schema.json create mode 100644 compass/extraction/data_centers/plugin_config.yaml create mode 100644 export_human_dc_ordinances.py create mode 100644 tests/python/unit/extraction/test_extraction_data_centers_schema.py create mode 100644 tests/python/unit/plugin/test_plugin_post_processing.py diff --git a/collapse_human_dc_ordinances.py b/collapse_human_dc_ordinances.py new file mode 100644 index 000000000..ed028c044 --- /dev/null +++ b/collapse_human_dc_ordinances.py @@ -0,0 +1,409 @@ +"""Collapse duplicate ordinance keys using the agreed feature-specific rules.""" + +import argparse +import csv +import re +from collections import Counter, defaultdict +from decimal import Decimal +from pathlib import Path + +# Features the extraction schema records as text rather than a number +QUALITATIVE = { + "parking", + "equipment screening/enclosure", + "fencing/security screening", + "backup generator operating restrictions", + "cooling technology", +} + +# Most restrictive value picked by hand from each multi-value row, keyed by +# (jurisdiction, feature). None marks a row with no single value to pick, +# such as a berm-only buffer or a setback that only scales with height. +MOST_RESTRICTIVE = { + ("Phoenix", "buffer/landscaping"): ("30", "feet"), + ("Elk Grove Village", "Minimum Lot Size"): ("4", "acres"), + ("Elk Grove Village", "Maximum Height"): ("100", "feet"), + ("Elk Grove Village", "Setback - Property Line"): ("50", "feet"), + ("Elk Grove Village", "Light Illumination Limits"): ("0.5", "footcandles"), + ("Elk Grove Village", "Maximum Lot Coverage"): ("50", "percent"), + ("Tippecanoe", "Vibration"): ("0.002", "g peak"), + ("Tippecanoe", "Setback - Property Line"): ("40", "feet"), + ("Tippecanoe", "Setback - Roads"): ("60", "feet"), + ("Tippecanoe", "Minimum Lot Size"): ("10000", "sq ft"), + ("Tippecanoe", "buffer/landscaping"): None, + ("Lancaster City", "buffer/landscaping"): ("100", "feet"), + ("West Pennsboro Township", "Minimum Lot Size"): ("25", "acres"), + ("West Pennsboro Township", "Setback - Property Line"): ("150", "feet"), + ("West Pennsboro Township", "Maximum Lot Coverage"): ("70", "percent"), + ("Loudoun", "Setback - Residential"): ("200", "feet"), + ("Loudoun", "buffer/landscaping"): None, + ("Prince William", "noise (low-frequency/tonal)"): ("68", "dB(C)"), + ("Prince William", "buffer/landscaping"): None, + ("Manassas City", "Setback - Property Line"): ("25", "feet"), + ("Manassas City", "Setback - Residential"): None, + ("Manassas City", "buffer/landscaping"): ("25", "feet"), + ("Albemarle", "noise mitigation/acoustic barriers"): ("70", "dBA"), +} + +# Units recorded with measurement detail, mapped to the unit alone. The +# detail moves into the summary, matching the extraction schema. +PLAIN_UNITS = { + "% max building coverage; no FAR standard": "percent", + "% minimum vegetative cover of lot": "percent", + "FAR (max)": "FAR", + "feet (max)": "feet", + "feet (up to)": "feet", + "feet from all lot lines": "feet", + "feet from listed receptors; measured property line to property line": "feet", + "feet from major/minor arterial; at least one arterial access required": "feet", + "feet from residential/community lot lines and residentially zoned or PO districts": "feet", + "feet from the Rural Areas zoning district": "feet", + "feet max building height": "feet", + "footcandles in residential or RE zone": "footcandles", + "percent annual cap": "percent", + "percent lot coverage": "percent", + "percent minimum reclaimed-water offset of total water use (director may set another percentage based on proximity to the reclaimed system)": "percent", + "sq ft GFA": "sq ft", + "sq ft/floor": "sq ft", + "°F maximum increase at zone boundary line": "°F", +} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "input", + nargs="?", + type=Path, + default=Path(__file__).with_name("human_extracted_dc_ordinances.csv"), + ) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + output = args.output or args.input.with_name( + "human_extracted_dc_ordinances_collapsed.csv" + ) + if output.resolve() == args.input.resolve(): + raise ValueError("Use a separate output file to preserve the original CSV.") + with args.input.open(encoding="utf-8-sig", newline="") as handle: + reader = csv.DictReader(handle) + columns = reader.fieldnames + original = list(reader) + groups = defaultdict(list) + for row in original: + groups[row_key(row)].append(row) + result = [] + for rows in groups.values(): + result.extend(collapse(rows) if len(rows) > 1 else rows) + validate(result, groups) + before = [row.copy() for row in result] + noise = select_lowest_noise_limits(result) + validate_noise(before, result) + before = [row.copy() for row in result] + reduced = select_most_restrictive_values(result) + validate_most_restrictive(before, result) + before = [row.copy() for row in result] + qualitative = use_summary_for_qualitative(result) + validate_qualitative(before, result) + before = [row.copy() for row in result] + filled = fill_value_from_summary(result) + validate_fill(before, result) + before = [row.copy() for row in result] + plain = use_plain_units(result) + validate_units(before, result) + with output.open("w", encoding="utf-8-sig", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=columns) + writer.writeheader() + writer.writerows(result) + with output.open(encoding="utf-8-sig", newline="") as handle: + reader = csv.DictReader(handle) + assert reader.fieldnames == columns + assert list(reader) == result + duplicates = sum(len(rows) > 1 for rows in groups.values()) + print( + f"{len(original)} -> {len(result)} rows; {duplicates} duplicate groups resolved." + ) + print(f"{noise} noise rows reduced to their lowest listed limit.") + print(f"{reduced} multi-value rows reduced to their most restrictive value.") + print(f"{qualitative} qualitative rows now carry their summary as value.") + print(f"{filled} rows had an empty value; copied their summary into value.") + print(f"{plain} rows had unit detail moved into their summary.") + print( + f"Validated unique keys, unchanged singleton rows, integer years, and CSV round-trip: {output}" + ) + + +def collapse(rows): + feature = rows[0]["feature"] + if feature in {"equipment screening/enclosure", "Cooling Technology"}: + row = merge_metadata(rows) + row.update(value=row["summary"], units="str") + return [row] + if feature == "Maximum Building Size": + assert len({r["units"] for r in rows}) == 1, ( + "Size limits must use comparable units." + ) + row = merge_metadata(rows) + row["value"] = str(min(Decimal(r["value"]) for r in rows)) + row["summary"] = ( + f"Selected most restrictive listed maximum: {row['value']} {row['units']}. " + + row["summary"] + ) + return [row] + if feature == "Noise": + # keep every listed limit; select_lowest_noise_limits picks one later + row = merge_metadata(rows) + row["value"] = "; ".join(r["value"].strip() for r in rows) + row["units"] = "; ".join(r["units"].strip() for r in rows) + return [row] + if feature == "definition": + return [collapse_definition(rows)] + raise ValueError(f"No collapse rule for duplicate feature: {feature}") + + +def select_lowest_noise_limits(rows): + """Keep the lowest listed limit as each Noise row's value. + + This matches the extraction schema, which records the single most + restrictive noise limit and leaves the other limits to the summary. + """ + selected = 0 + for row in rows: + if row["feature"] != "Noise": + continue + limit = min(listed_noise_limits(row["value"])) + row["summary"] = ( + f"Selected most restrictive listed limit: {limit} dBA " + f"(listed: {row['value']} {row['units']}). " + row["summary"] + ) + row.update(value=str(limit), units="dBA") + selected += 1 + return selected + + +def listed_noise_limits(value): + """Numbers in a noise value, skipping '+N' exception allowances. + + A segment such as '+10 for 15 min' raises the limit for a short + duration rather than setting one, so none of its numbers are limits. + """ + return [ + Decimal(number) + for segment in value.split(";") + if not segment.strip().startswith("+") + for number in re.findall(r"\d+(?:\.\d+)?", segment) + ] + + +def validate_noise(before, after): + """Only Noise rows changed, each to the lowest of its listed limits.""" + assert len(before) == len(after) + for old, new in zip(before, after, strict=True): + if old["feature"] != "Noise": + assert new == old, "A non-noise row was changed." + continue + assert Decimal(new["value"]) == min(listed_noise_limits(old["value"])) + assert new["units"] == "dBA" + assert new["summary"].endswith(old["summary"]) + rewritten = {"value", "units", "summary"} + assert all(new[k] == v for k, v in old.items() if k not in rewritten) + + +def collapse_definition(rows): + """Keep the adopted definition primary and label denied text as historical.""" + adopted = [row for row in rows if "ADOPTED/CURRENT." in row["explanation"]] + assert len(adopted) == 1, "A single adopted definition must be identified." + row = adopted[0].copy() + historical = [r for r in rows if r is not adopted[0]] + assert all("PROPOSED/DENIED." in r["explanation"] for r in historical) + row["summary"] = ( + "Uses the adopted July 7, 2026 moratorium Resolution definition, which draws no " + "size distinctions and applies for the moratorium period through March 30, 2027, " + "as recorded in the workbook. The denied 2026 draft definition is retained in " + "explanation as historical text and is not part of the selected definition." + ) + notes = ["Original adopted-definition notes:\n" + row["explanation"].strip()] + for old in historical: + notes.append( + "Historical proposed/denied definition (not operative):\n" + + old["value"] + + f"\nSection: {old['section']}\nYear: {old['year']}\nSource: {old['source']}" + + "\nOriginal denied-draft notes:\n" + + old["explanation"].strip() + ) + row.update(explanation="\n\n".join(notes), units="str") + return row + + +def select_most_restrictive_values(rows): + """Reduce each multi-value row to the value picked in MOST_RESTRICTIVE. + + The listed values are kept at the front of the summary, matching the + extraction schema, which records one value and leaves the rest to the + summary. + """ + matched = set() + for row in rows: + key = (row["subdivision"].strip() or row["county"].strip(), row["feature"]) + if key not in MOST_RESTRICTIVE: + continue + assert key not in matched, f"More than one row for {key}" + matched.add(key) + listed = f"(listed: {row['value']} {row['units']})" + picked = MOST_RESTRICTIVE[key] + if picked is None: + row["summary"] = f"No single most restrictive value {listed}. " + row["summary"] + row.update(value=row["summary"], units="str") + else: + value, units = picked + row["summary"] = ( + f"Selected most restrictive listed value: {value} {units} {listed}. " + + row["summary"] + ) + row.update(value=value, units=units) + assert matched == set(MOST_RESTRICTIVE), set(MOST_RESTRICTIVE) - matched + return len(matched) + + +def validate_most_restrictive(before, after): + """Only picked rows changed, each to a number its old value listed.""" + assert len(before) == len(after) + for old, new in zip(before, after, strict=True): + if new == old: + continue + key = (old["subdivision"].strip() or old["county"].strip(), old["feature"]) + assert key in MOST_RESTRICTIVE, f"An unpicked row was changed: {key}" + assert new["summary"].endswith(old["summary"]) + if MOST_RESTRICTIVE[key] is not None: + assert new["value"] in old["value"].replace(",", "") + rewritten = {"value", "units", "summary"} + assert all(new[k] == v for k, v in old.items() if k not in rewritten) + + +def use_plain_units(rows): + """Keep only the unit in units and move its detail into the summary.""" + changed = 0 + for row in rows: + units = row["units"].strip() + if units not in PLAIN_UNITS: + continue + row["summary"] = f"Units detail: {units}. " + row["summary"] + row["units"] = PLAIN_UNITS[units] + changed += 1 + return changed + + +def validate_units(before, after): + """Only detailed units changed, and every numeric row has a plain unit.""" + assert len(before) == len(after) + for old, new in zip(before, after, strict=True): + if new == old: + continue + units = old["units"].strip() + assert new == { + **old, + "units": PLAIN_UNITS[units], + "summary": f"Units detail: {units}. " + old["summary"], + } + allowed = set(PLAIN_UNITS.values()) | {"str", "acres", "dBA", "dB(C)", "g peak"} + assert all(row["units"] in allowed for row in after), { + row["units"] for row in after + } - allowed + + +def use_summary_for_qualitative(rows): + """Carry the summary as value for features the schema extracts as text. + + This follows the convention finalize applies to qualitative features. + """ + changed = 0 + for row in rows: + if row["feature"].casefold() not in QUALITATIVE: + continue + summary = row["summary"].strip() + if summary and (row["value"], row["units"]) != (summary, "str"): + row.update(value=summary, units="str") + changed += 1 + return changed + + +def validate_qualitative(before, after): + """Only qualitative rows changed, and only their value and units.""" + assert len(before) == len(after) + for old, new in zip(before, after, strict=True): + if new == old: + continue + assert old["feature"].casefold() in QUALITATIVE + assert new == {**old, "value": old["summary"].strip(), "units": "str"} + + +def fill_value_from_summary(rows): + """Use the summary as the value when the value is empty but a summary exists. + + Free-text answers are sometimes recorded only in ``summary``; an empty + ``value`` would otherwise read as "nothing found" downstream. + """ + filled = 0 + for row in rows: + if not row["value"].strip() and row["summary"].strip(): + row["value"] = row["summary"].strip() + filled += 1 + return filled + + +def validate_fill(before, after): + """Only empty values with a summary changed, and only their value.""" + assert len(before) == len(after) + for old, new in zip(before, after, strict=True): + if old["value"].strip() or not old["summary"].strip(): + assert new == old, "A row with a value (or no summary) was changed." + else: + assert new == {**old, "value": old["summary"].strip()} + assert all(row["value"].strip() for row in after if row["summary"].strip()) + + +def merge_metadata(rows): + row = rows[0].copy() + assert len({r["year"] for r in rows}) == 1 + for field in ("section", "summary", "explanation", "source"): + separator = " " if field == "summary" else "\n\n" + row[field] = separator.join( + dict.fromkeys(r[field].strip() for r in rows if r[field].strip()) + ) + return row + + +def validate(result, groups): + counts = Counter(row_key(row) for row in result) + assert all(count == 1 for count in counts.values()) + assert all(not row["year"] or row["year"].isdigit() for row in result) + assert all( + row["data_center_type"].lower() not in {"all", "data center"} for row in result + ) + by_key = {row_key(row): row for row in result} + for key, rows in groups.items(): + if len(rows) == 1: + assert by_key[key] == rows[0], "A singleton row was changed." + elif rows[0]["feature"] in { + "equipment screening/enclosure", + "Cooling Technology", + }: + merged = by_key[key] + assert merged["units"] == "str" and merged["value"] + assert all(row["summary"].strip() in merged["value"] for row in rows) + assert len(result) == len(groups) + + +def row_key(row): + fields = ( + "state", + "county", + "subdivision", + "jurisdiction_type", + "data_center_type", + "feature", + ) + return tuple(row[field].strip().casefold() for field in fields) + + +if __name__ == "__main__": + main() diff --git a/compass/__init__.py b/compass/__init__.py index 9af98f7e8..8ccb7a77a 100644 --- a/compass/__init__.py +++ b/compass/__init__.py @@ -8,6 +8,7 @@ # Temporarily import to register plugins # Can drop once plugins register themselves from .extraction import ( + COMPASSDataCentersExtractor, COMPASSGeoHeatPumpExtractor, COMPASSGeoElectricityExtractor, COMPASSNaturalGasPipelinesExtractor, diff --git a/compass/extraction/__init__.py b/compass/extraction/__init__.py index 6bcb6a4a6..fb0607b17 100644 --- a/compass/extraction/__init__.py +++ b/compass/extraction/__init__.py @@ -10,6 +10,7 @@ # Temporarily import to register plugins # Can drop once plugins register themselves +from .data_centers import COMPASSDataCentersExtractor from .ghp import COMPASSGeoHeatPumpExtractor from .geothermal_electricity import COMPASSGeoElectricityExtractor from .natural_gas import COMPASSNaturalGasExtractor diff --git a/compass/extraction/data_centers/__init__.py b/compass/extraction/data_centers/__init__.py new file mode 100644 index 000000000..a670aa41c --- /dev/null +++ b/compass/extraction/data_centers/__init__.py @@ -0,0 +1,12 @@ +"""COMPASS data centers plugin""" + +import importlib.resources + +from compass.plugin import create_schema_based_one_shot_extraction_plugin + + +COMPASSDataCentersExtractor = create_schema_based_one_shot_extraction_plugin( + importlib.resources.files("compass.extraction.data_centers") + / "plugin_config.yaml", + tech="data_centers", +) diff --git a/compass/extraction/data_centers/data_centers_schema.json b/compass/extraction/data_centers/data_centers_schema.json new file mode 100644 index 000000000..7f8622820 --- /dev/null +++ b/compass/extraction/data_centers/data_centers_schema.json @@ -0,0 +1,325 @@ +{ + "title": "Data Center Ordinance Extraction Schema", + "description": "One-shot extraction schema for local data center siting ordinances. Feature IDs mirror the controlled vocabulary used in the COMPASS data center ordinance research workbook, so extracted rows can be compared directly against the hand-coded reference set. The output is a sparse long-form table where each object is one row.", + "version": "2.0.0", + "type": "object", + "required": [ + "outputs", + "extraction_notes" + ], + "additionalProperties": false, + "properties": { + "outputs": { + "type": "array", + "description": "Sparse long-form extraction table. Emit a row only for a feature with an explicit, enacted requirement. Emit at most one row per (feature, data center class) pair: where an ordinance sets a different standard for each size class of data center, emit one row per class rather than collapsing them.", + "items": { + "type": "object", + "required": [ + "feature", + "data_center_type", + "value", + "units", + "section", + "summary", + "ordinance_text", + "explanation" + ], + "additionalProperties": false, + "properties": { + "feature": { + "type": "string", + "description": "The ordinance feature being extracted. Must be one of the enumerated feature IDs. Do not invent aliases, prefixes, or synonym variants.", + "enum": [ + "setback - residential", + "setback - property line", + "setback - roads", + "setback - transit", + "setback - parks/natural areas", + "setback - data centers", + "setback for mechanical equipment", + "minimum lot size", + "maximum lot coverage", + "minimum vegetative cover", + "maximum height", + "maximum building size", + "data center floor-area share cap", + "parking", + "primary use districts", + "special use districts", + "accessory use districts", + "prohibited use districts", + "prohibitions/moratorium", + "noise", + "noise (low-frequency/tonal)", + "noise mitigation/acoustic barriers", + "vibration", + "light illumination limits", + "heat/thermal impact", + "buffer/landscaping", + "equipment screening/enclosure", + "fencing/security screening", + "backup generator operating restrictions", + "on-site power technology", + "decommissioning", + "cooling technology", + "water consumption limit", + "water reuse/reclaimed", + "definition" + ] + }, + "data_center_type": { + "type": [ + "string", + "null" + ], + "description": "The specific facility class governed by the provision, such as 'Hyperscale Data Center', 'Micro Data Center', or a jurisdiction-specific size class. Use null when the requirement applies to data centers generally, including generic terms such as 'Data Center', 'DATA CENTER', or 'Data Processing Center'. For accessory data centers, use 'Accessory Data Center' without parenthetical conditions. Keep conditions in summary and preserve the complete source wording in ordinance_text." + }, + "value": { + "description": "The extracted requirement. For a numeric feature, a single number: the most restrictive value the ordinance sets for this feature and data center class, chosen by the rules in $instructions.numeric_prioritization. Never combine several numbers into one value; the other values belong in summary. Array of district names/codes for district features, boolean true where a requirement exists but is not numeric, or null where the feature is qualitative and the substance lives in the summary.", + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "boolean" + }, + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "null" + } + ] + }, + "units": { + "type": [ + "string", + "null" + ], + "description": "The unit of the value, written with the canonical vocabulary in $instructions.units, such as 'feet', 'acres', 'dBA', or 'percent'. Never convert the value into a different unit. Keep this to the unit alone: the measurement point, time period, and receiving use belong in summary. Null for district, qualitative, and other features that carry no unit." + }, + "section": { + "type": [ + "string", + "null" + ], + "description": "Pin cite to the governing provision, as specific as the source allows (chapter, article, section, and subsection). Null only when the source text carries no section identifier." + }, + "summary": { + "type": [ + "string", + "null" + ], + "description": "A short summary of the requirement in your own words, retaining all specifics and conditions, such as an exception for a power outage, a standard that steps up next to a residential district, or a sunset date. Where the ordinance sets several values for this feature, state which one value holds and list every other value with the condition it applies under, such as the time period, receiving use, zoning district, street class, or yard. The verbatim excerpt belongs in ordinance_text. For qualitative features, state the requirement in full." + }, + "ordinance_text": { + "type": "string", + "description": "The complete relevant text excerpt copied verbatim from the source document. Start with the full sentence corresponding to the extracted value and summary, without elision. Then include other passages supporting the same requirement, in source order, joining separate passages with an ellipsis ('...'). Preserve the facility class, conditions, and exceptions that govern this row. Every character outside the ellipses must appear exactly as written in the source. Do not paraphrase, normalize, or add commentary. Must be a non-empty string." + }, + "explanation": { + "type": "string", + "description": "Brief rationale explaining why the evidence in ordinance_text matches the selected feature and data_center_type, and how it supports the extracted value and units or the inclusion of a qualitative requirement. Must be a non-empty string." + } + } + } + }, + "extraction_notes": { + "type": "string", + "description": "One or two sentences on what was and was not extracted, and why. Always fill this in. When outputs is empty, state why nothing qualified (for example: the text has no data center provisions, only proposed or superseded provisions, or a moratorium whose end date has passed). When relevant text was deliberately left out of outputs, name it and give the reason." + } + }, + "$definitions": { + "setbacks": { + "setback - residential": { + "description": "Minimum separation between a data center and a residential or otherwise protected receptor, measured to a residential lot line, a residential zoning district boundary, or a property in residential use. Includes rules that name civic receptors such as schools, parks, churches, day care, or hospitals alongside residential property, and height-scaled setback increases triggered by adjacency to residential property. VALUE: the largest minimum distance. UNITS: a distance unit from $instructions.units, such as 'feet'. SUMMARY: name the receptors, any height-scaled increase with its trigger height, and the distances that apply to other receptors or districts. IGNORE: generic lot-line standards whose subject is the district boundary rather than a residential receptor, which belong under 'setback - property line'." + }, + "setback - property line": { + "description": "Minimum setback measured to a lot line, yard boundary, or zoning district boundary where the adjoining use is not residential or otherwise protected. Includes district front, side, and rear yard standards, and standards that step up where a lot abuts a residential district, since there the base requirement is generic and the residential case is a modifier rather than the subject. VALUE: the largest minimum distance across the front, side, rear, and per-district standards. UNITS: a distance unit from $instructions.units, such as 'feet'. SUMMARY: list the other yard and district distances, and any step-up next to a residential district, with their conditions. IGNORE: rules whose stated purpose is separation from a residential or protected receptor, which belong under 'setback - residential'." + }, + "setback - roads": { + "description": "Minimum setback measured to a roadway, street frontage, or right-of-way, including standards that scale by roadway classification. VALUE: the largest minimum distance. UNITS: a distance unit from $instructions.units, such as 'feet'. SUMMARY: list the distances for the other street classes. IGNORE: access, driveway spacing, and traffic study requirements that impose no separation distance." + }, + "setback - transit": { + "description": "Minimum separation between a data center and a high-capacity transit station, stop, or facility, including fixed guideway, bus rapid transit, commuter rail, and metro stations, measured to the transit facility. These provisions typically exist to reserve transit-served land for higher-employment uses rather than to protect the transit facility from impacts. VALUE: the largest minimum distance. UNITS: a distance unit from $instructions.units, such as 'feet'. IGNORE: transit-oriented development policy language that states no separation distance." + }, + "setback - parks/natural areas": { + "description": "Minimum separation from public parkland, trails, riparian corridors, natural areas, or designated open space. VALUE: the largest minimum distance. UNITS: a distance unit from $instructions.units, such as 'feet'. SUMMARY: note where the ordinance frames the distance as a maximum applied at the reviewer's discretion. IGNORE: environmental review, habitat study, and tree preservation requirements that state no separation distance." + }, + "setback - data centers": { + "description": "Minimum separation between a data center or data center campus and another data center. VALUE: the largest minimum distance. UNITS: a distance unit from $instructions.units, such as 'feet'. IGNORE: aggregation rules that define when adjacent buildings count as one campus without imposing a separation distance." + }, + "setback for mechanical equipment": { + "description": "Minimum setback applying to ancillary mechanical or electrical equipment such as generators, transformers, substations, chillers, cooling and ventilation equipment, or battery storage, rather than to the data center building. Code this feature only where the ordinance holds equipment to a different standard than the building; where one distance covers both, record the row under the applicable reference-object setback feature instead. VALUE: the largest minimum distance. UNITS: a distance unit from $instructions.units, such as 'feet'. SUMMARY: name the equipment and the receptor the distance is measured to. IGNORE: screening and enclosure requirements for the same equipment, which belong under 'equipment screening/enclosure'." + } + }, + "site_dimensions": { + "minimum lot size": { + "description": "Minimum lot or site area a data center must occupy, which may differ by zoning district. A floor on how much land the use requires. VALUE: the largest minimum lot area. UNITS: an area unit from $instructions.units, such as 'acres' or 'sq ft'. SUMMARY: list any minimum lot width and the areas that apply in other districts. IGNORE: ceilings on how much of that land may be built or paved, which belong under 'maximum lot coverage'." + }, + "maximum lot coverage": { + "description": "Ceiling on how much of a data center site may be built or paved, expressed as maximum building coverage, maximum lot coverage, maximum impervious coverage, or floor area ratio. VALUE: the lowest maximum coverage percentage, or the lowest floor area ratio where the ordinance states only a floor area ratio. UNITS: 'percent' or 'FAR'. SUMMARY: list any floor area ratio stated alongside a coverage percentage, impervious coverage limits, and the limits that apply in other districts. IGNORE: minimum land area requirements ('minimum lot size') and planting obligations ('minimum vegetative cover')." + }, + "minimum vegetative cover": { + "description": "Minimum share of a data center site that must be planted or retained in vegetation. A planting obligation rather than a limit on construction: it is not the complement of maximum coverage, since the remainder of a site may be pavement or hardstanding. VALUE: the largest minimum share. UNITS: 'percent'. SUMMARY: the basis the ordinance measures against, such as the lot. IGNORE: perimeter screening keyed to adjacent uses, which belongs under 'buffer/landscaping'." + }, + "maximum height": { + "description": "Maximum permitted height of a data center structure, expressed in feet, in stories, or in both, and often differing by zoning district. VALUE: the lowest maximum height, in feet where the ordinance states feet. UNITS: 'feet', or 'stories' where the ordinance states only stories. SUMMARY: list any story limit and the heights that apply in other districts. IGNORE: height-triggered setback increases, which belong on the applicable setback feature." + }, + "maximum building size": { + "description": "Maximum permitted data center building size, expressed as gross floor area or as square feet per floor. VALUE: the lowest maximum. UNITS: an area unit from $instructions.units, such as 'acres' or 'sq ft'. SUMMARY: state whether the cap is on gross floor area or per floor. IGNORE: floor area thresholds that merely trigger a permit pathway rather than capping building size, and share-of-project caps, which belong under 'data center floor-area share cap'." + }, + "data center floor-area share cap": { + "description": "Maximum data center share of total project floor area, including any occupancy-sequencing condition imposed on the project's other uses. VALUE: the lowest maximum share. UNITS: 'percent'. SUMMARY: any occupancy-sequencing condition. IGNORE: absolute floor area caps, which belong under 'maximum building size'." + }, + "parking": { + "description": "Off-street parking requirement for a data center use, expressed as a numerical ratio, minimum, maximum, or distance standard. The base differs by jurisdiction (total building area, office floor area only, or employee count), so ratios are not comparable across jurisdictions. VALUE: null; the substance belongs in the summary. UNITS: null. SUMMARY: record each ratio or count with the base it is measured against, and any threshold above which a different ratio applies. IGNORE: loading, stacking, and bicycle parking standards." + } + }, + "districts_and_permitting": { + "primary use districts": { + "description": "Zoning districts in which data centers are permitted as a primary use, that is without special conditions or discretionary approval. Following the COMPASS wind and solar convention, overlay districts are treated as primary use districts. VALUE: array of district names or codes, carrying the jurisdiction's own permit abbreviation alongside the district names where the ordinance uses one. UNITS: null. SUMMARY: any stated threshold that conditions the by-right pathway. IGNORE: districts where the use requires discretionary approval, which belong under 'special use districts'." + }, + "special use districts": { + "description": "Zoning districts in which data centers are permitted only with discretionary approval, whether by special use permit, special exception, conditional use, provisional use permit, special land use permit, development agreement, or an equivalent. VALUE: array of district names or codes, carrying the jurisdiction's own permit abbreviation alongside the district names. UNITS: null. SUMMARY: the approval pathway, naming the reviewing bodies where the ordinance does. IGNORE: generic permit administration and fee schedules not tied to data centers." + }, + "accessory use districts": { + "description": "Zoning districts in which a data center is permitted as an accessory or secondary use to a principal use rather than as a principal use in its own right. VALUE: array of district names or codes, carrying the jurisdiction's own permit abbreviation alongside the district names, or the ordinance's own phrasing where it permits the accessory use across all zones. UNITS: null. IGNORE: districts where the use is permitted as a principal use." + }, + "prohibited use districts": { + "description": "Zoning districts in which data centers are prohibited outright. VALUE: array of district names or codes, or the ordinance's own phrasing where a district list is stated as having no pathway. UNITS: null. IGNORE: jurisdiction-wide bans, moratoria, and use removals, which belong under 'prohibitions/moratorium'." + }, + "prohibitions/moratorium": { + "description": "Temporary moratorium, permanent prohibition, or removal of data centers as an allowed use, applying to the jurisdiction as a whole rather than to a list of districts. VALUE: null; the substance belongs in the summary. UNITS: null. SUMMARY: state the instrument, what it bars, and the effective and expiration dates where given, along with any carve-out such as a limited accessory allowance. IGNORE: district-level prohibitions, which belong under 'prohibited use districts'." + } + }, + "noise_and_nuisance": { + "noise": { + "description": "Maximum operational noise limits, usually expressed in dBA and often separated by time period, by receiving use, or by the zoning district of the receiving property. VALUE: the lowest numeric limit across all time periods, receiving uses, and districts, which is usually the nighttime limit at residential property. UNITS: 'dBA'. SUMMARY: list every other limit with its time period and receiving use or district, the measurement location, and any exception, such as one for a power outage or a stated allowance for a short duration. IGNORE: complaint-response protocol, monitoring workflow, and report submission timelines that state no numeric limit." + }, + "noise (low-frequency/tonal)": { + "description": "Numerical C-weighted, low-frequency, or tonal sound limits, which are distinct from conventional A-weighted dBA limits. VALUE: the lowest numeric limit across all time periods and receptors. UNITS: 'dB(C)'. SUMMARY: list the other limits with their time periods, and the measurement location and interval. IGNORE: A-weighted limits, which belong under 'noise'." + }, + "noise mitigation/acoustic barriers": { + "description": "Numerical noise attenuation standard for an enclosure or barrier serving mechanical equipment or generators, expressed as a maximum sound level measured at a stated distance. VALUE: the lowest maximum sound level. UNITS: 'dBA'. SUMMARY: the distance at which the level is measured and the equipment the standard covers. IGNORE: ambient and property line noise limits, which belong under the noise features, and non-numeric enclosure requirements, which belong under 'equipment screening/enclosure'." + }, + "vibration": { + "description": "Numerical vibration limit, duration threshold, or setback applying to vibration-producing equipment, measured at a property boundary or other receptor. VALUE: the lowest numeric vibration limit, or the largest setback distance where the ordinance states only a setback. UNITS: 'g peak', or 'feet' for a setback. SUMMARY: the receptor and any excluded duration. IGNORE: construction-period vibration provisions that do not govern operation." + }, + "light illumination limits": { + "description": "Maximum illumination received at a property line, lot line, zoning district boundary, or other receptor, expressed in foot-candles, lux, or an equivalent unit. Measures light crossing onto a neighboring parcel rather than glare as visual intrusion. VALUE: the lowest numeric maximum. UNITS: 'footcandles' or 'lux'. SUMMARY: the receptor the limit is measured at, and the limits that apply at other receiving districts. IGNORE: dark-sky standards such as uplight caps and correlated color temperature limits, which govern skyglow rather than illumination at a receptor." + }, + "heat/thermal impact": { + "description": "Numerical ambient temperature or thermal impact limit measured at a zoning boundary, property line, or other receptor. VALUE: the lowest numeric maximum, stated as an increase where the ordinance frames it that way. UNITS: '°F' or '°C'. SUMMARY: the receptor and whether the limit is an increase over ambient. IGNORE: waste heat recovery and heat reuse encouragement that imposes no limit." + } + }, + "site_design": { + "buffer/landscaping": { + "description": "Numerical buffer widths, berm or fence dimensions, planting quantities, spacing, composition, coverage, or installation requirements at the site perimeter. VALUE: the largest minimum buffer width, or null where the ordinance sets no buffer width. UNITS: a distance unit from $instructions.units, such as 'feet', or null with a null value. SUMMARY: list the other buffer widths with their conditions, and the berm or fence dimensions, planting quantities, spacing, composition, and coverage standards. IGNORE: requirements to screen mechanical equipment, which belong under 'equipment screening/enclosure', and site-wide planting shares, which belong under 'minimum vegetative cover'." + }, + "equipment screening/enclosure": { + "description": "Requirement to enclose or visually screen mechanical and electrical equipment serving a data center, including rooftop and ground-mounted units, generators, substations, loading and service areas, and refuse areas. VALUE: null; the substance belongs in the summary. UNITS: null. SUMMARY: what must be screened or enclosed and any numeric standard for wall or screen height, opacity, articulation, or clearance above the equipment. IGNORE: perimeter planting ('buffer/landscaping') and perimeter fencing ('fencing/security screening')." + }, + "fencing/security screening": { + "description": "Perimeter fencing or security screening requirement, including permitted and prohibited fence types and any height, opacity, or setback standard. VALUE: null; the substance belongs in the summary. UNITS: null. SUMMARY: the required or prohibited fence types and any height, stating whether it is a minimum or a maximum, opacity, or setback standard. IGNORE: screening of mechanical equipment, which belongs under 'equipment screening/enclosure', and security operations that impose no fencing requirement." + } + }, + "power_and_operations": { + "backup generator operating restrictions": { + "description": "Limits on when, and for how long, an emergency or backup generator may be operated: permitted testing and maintenance windows, maximum hours per day or month, maximum consecutive run time, and carve-outs for power outages or utility demand-response events. VALUE: null; the substance belongs in the summary. UNITS: null. SUMMARY: the permitted windows and the days and seasons they apply to, any hour limits, and the carve-outs. IGNORE: emissions standards, public notice and disclosure obligations, utility interconnection requirements, compliance logging, acoustic enclosure standards, and backup energy technology mandates, the last of which belongs under 'on-site power technology'." + }, + "on-site power technology": { + "description": "Required or prohibited on-site electricity generation and backup energy technology, including which generation types may serve as a primary, parallel, or routine source, which fuel types are permitted for backup service, any mandated battery or energy storage backup, and dispatch priority between sources. VALUE: null; the substance belongs in the summary. UNITS: null. IGNORE: limits on when generators may run, which belong under 'backup generator operating restrictions', and battery energy storage fire safety provisions." + }, + "decommissioning": { + "description": "End-of-life or decommissioning obligation for a data center, including equipment removal, disposal or recycling of servers and electronic waste, notification on cessation or abandonment, financial security, site restoration, and any timing requirement for beginning or completing decommissioning. VALUE: null; the substance belongs in the summary. UNITS: null. SUMMARY: record the triggering event and any stated deadline or security amount." + } + }, + "water_and_cooling": { + "cooling technology": { + "description": "Required or prohibited cooling technology, such as closed-loop, air-cooled, recycled-water, or no potable-water cooling. VALUE: null; the substance belongs in the summary. UNITS: null. SUMMARY: record which technologies are required or prohibited, any water use intensity stated for cooling with its units, and any circumstance under which potable water remains allowed, such as initial system charging or an emergency. IGNORE: volume ceilings ('water consumption limit') and reclaimed-source requirements ('water reuse/reclaimed')." + }, + "water consumption limit": { + "description": "Enforceable numerical ceiling on the volume of water a data center may use, expressed as a cap on annual or periodic consumption. VALUE: the lowest numeric ceiling. UNITS: a volume unit from $instructions.units, such as 'gallons', or 'percent' where the cap is expressed relative to a baseline allocation. SUMMARY: the period the cap covers and any baseline it is measured against. IGNORE: water conservation and management plans, which require a document rather than observance of a limit, and efficiency intensity standards, which set an intensity rather than a volume ceiling." + }, + "water reuse/reclaimed": { + "description": "Requirement or incentive to substitute reclaimed, recycled, or non-potable water for potable supply, including any minimum share of total water use that must be met from reclaimed sources. Governs the source of water rather than the volume. VALUE: the largest minimum share where stated, otherwise boolean true where reuse is required without a numeric share. UNITS: 'percent', or null with a boolean value. SUMMARY: the basis of the share and any administrative discretion to set a different share. IGNORE: volume ceilings, which belong under 'water consumption limit'." + } + }, + "terms": { + "definition": { + "description": "The jurisdiction's definition of a data center term, such as 'Data Center', 'Data Processing Center', or a size class like 'Hyperscale Data Center'. Emit one row per defined term, with data_center_type set to the defined class name, or null for the generic data center term. VALUE: the definition text, verbatim. UNITS: null. SECTION: where the term is defined. IGNORE: definitions of other uses that merely mention data processing or computing in passing, such as a business office that lists data processing among its activities." + } + } + }, + "$instructions": { + "general": [ + "Extract only enacted local requirements that explicitly govern data centers, or that govern a use the ordinance defines as a data center, such as a data processing center or a computer server facility.", + "Also record each definition of a data center term as a 'definition' row, even though a definition is not itself a requirement.", + "Write summary in your own words. Put verbatim source passages in ordinance_text and explain how they support the row in explanation.", + "Never convert units. Preserve the jurisdiction's vocabulary except for the data_center_type rules and the canonical unit names. Keep ordinance_text verbatim.", + "Do not infer a requirement from related context, and do not emit a row for a feature the source does not address." + ], + "size_tiering": [ + "Data center ordinances frequently set different standards for different size classes of facility. When they do, emit one row per class and record the jurisdiction's own class name in data_center_type.", + "Where one standard applies to data centers generally, emit a single row with data_center_type set to null. Generic labels such as 'Data Center' or 'DATA CENTER' must not appear as a specific facility class.", + "Do not merge tiered standards into a single row, and do not map a local class name onto another jurisdiction's taxonomy.", + "For accessory data centers, use 'Accessory Data Center' without parenthetical conditions. Retain those conditions in summary and preserve them verbatim in ordinance_text." + ], + "setbacks": [ + "Setbacks are minimum separation distances. A setback row carries a single numeric distance and its unit.", + "Decide between 'setback - residential' and 'setback - property line' by what the rule is for: a rule whose subject is separation from a residential or protected receptor is residential, while a generic district yard standard that merely steps up next to a residential district is a property line setback.", + "Where a setback applies to ancillary mechanical or electrical equipment under a different standard than the building, record it under 'setback for mechanical equipment'; where one distance covers both, keep it on the reference-object feature." + ], + "numeric_prioritization": [ + "A numeric feature gets one row per data center class, and its value is a single number: the most restrictive value the ordinance sets for that feature and class.", + "Restrictiveness rules: setbacks, minimum lot size, minimum vegetative cover, buffer width, and minimum reclaimed water share -> choose the largest value; noise, low-frequency noise, enclosure sound levels, vibration, illumination, thermal increase, maximum height, maximum building size, maximum lot coverage, floor-area share cap, and water consumption ceilings -> choose the smallest value.", + "Values that differ by zoning district, time period, receiving use, street class, or yard compete for the same row. Do not emit a row per condition and do not combine the values into one string. List every value that was not selected in summary, with the condition it applies under.", + "Size classes are the exception: standards for different data center classes stay in separate rows, per size_tiering, and the most restrictive value is chosen within each class.", + "Compare values across units if needed to find the most restrictive one, but record the selected value in its own unit. Never convert the reported value." + ], + "units": [ + "Write units with this canonical vocabulary: 'feet', 'miles', 'meters', 'acres', 'sq ft', 'stories', 'percent', 'FAR', 'dBA', 'dB(C)', 'footcandles', 'lux', 'g peak', '°F', '°C', 'gallons', 'acre-feet'. Spell the unit this way even where the ordinance abbreviates it: 'ft' becomes 'feet', 'ac' becomes 'acres', and 'dB(A)' becomes 'dBA'.", + "Choosing the canonical spelling is not a conversion. Never convert a value into a different unit: a distance stated in meters stays in meters.", + "Where the ordinance uses a unit not on the list, write its full name in lowercase.", + "Units hold the unit alone. Measurement points, time periods, receiving uses, approval pathways, and qualifiers such as 'maximum' or 'up to' belong in summary." + ], + "district_specific": [ + "For primary use districts, extract an array of district names or codes and treat overlay districts as primary use.", + "For special use districts, include districts where the use requires a special use permit, special exception, conditional use, or an equivalent discretionary approval.", + "For accessory use districts, include districts where a data center is allowed only as accessory or secondary to a principal use.", + "For prohibited use districts, include only districts where the use is barred outright and the prohibition is currently in effect.", + "Carry the jurisdiction's own permit abbreviation alongside the district names, and use district names and codes exactly as they appear in the ordinance." + ], + "null_handling": "Use sparse output. Emit rows only for features explicitly addressed by the source. Do not emit placeholder rows for missing features. Use a null value where a present feature is qualitative and the substance belongs in the summary." + }, + "$qa_checklist": [ + "All output rows use only the 35 allowed feature IDs.", + "Every row includes a non-empty ordinance_text excerpt and an explanation linking that evidence to the feature and facility class.", + "Generic requirements have null data_center_type. Specific facility classes retain their names, tiered standards have one row per class, and accessory types use 'Accessory Data Center' without parenthetical conditions.", + "Numeric setbacks, lot standards, and building size rows carry numeric values and non-null units.", + "Numeric rows carry a single number, the most restrictive value, with a plain unit, and summary lists the other values with their conditions.", + "Units use only the canonical vocabulary in $instructions.units and carry no measurement detail.", + "Noise rows carry the lowest explicit numeric limit, summary lists the other limits with their time periods and receiving uses, and no row is emitted from complaint-protocol language alone.", + "District rows are arrays of district names or codes, and jurisdiction-wide bans are recorded under 'prohibitions/moratorium' rather than 'prohibited use districts'.", + "Screening, buffer, and fencing rows are assigned by what the provision governs: equipment, site perimeter planting, or perimeter fencing.", + "Water rows are assigned by what the provision governs: technology, volume ceiling, or source substitution.", + "No row is emitted from a proposed, denied, or superseded provision." + ], + "$qualitative_features": [ + "parking", + "prohibitions/moratorium", + "equipment screening/enclosure", + "fencing/security screening", + "backup generator operating restrictions", + "on-site power technology", + "decommissioning", + "cooling technology", + "definition" + ] +} diff --git a/compass/extraction/data_centers/plugin_config.yaml b/compass/extraction/data_centers/plugin_config.yaml new file mode 100644 index 000000000..69e3f9901 --- /dev/null +++ b/compass/extraction/data_centers/plugin_config.yaml @@ -0,0 +1,164 @@ +schema: ./data_centers_schema.json + +data_type_short_desc: data center ordinance + +cache_llm_generated_content: true + +post_processing_steps: + - normalize_data_center_types + +query_templates: + - "{jurisdiction} data center ordinance" + # Recently adopted ordinances are often published only as a PDF on the + # jurisdiction's own site, before they are codified in a code library. + - "filetype:pdf {jurisdiction} data center ordinance" + - "site:{jurisdiction_website} data center" + - "{jurisdiction} zoning ordinance" + +website_keywords: + pdf: 92160 + "data center": 46080 + datacenter: 46080 + ordinance: 23040 + zoning: 11520 + setback: 5760 + noise: 5760 + "conditional use": 2880 + "special use": 2880 + hyperscale: 1440 + colocation: 1440 + generator: 1440 + "code of ordinances": 1440 + "municipal code": 1440 + ecode360: 1440 + municode: 1440 + "zoning ordinance": 1440 + "unified development ordinance": 720 + "ordinance adopted": 720 + "land use code": 720 + "text amendment": 720 + "land development": 720 + cooling: 360 + screening: 360 + county: 360 + municipal: 360 + chapter: 180 + section: 180 + planning: 180 + +heuristic_keywords: + GOOD_TECH_KEYWORDS: + - datacenter + - hyperscale + - colocation + GOOD_TECH_PHRASES: + - data center + - data processing center + - data processing facility + - data center campus + - computer server + - server farm + - colocation facility + - backup generator + - emergency generator + - zoning ordinance + - code of ordinances + - municipal code + - unified development ordinance + - conditional use permit + GOOD_TECH_ACRONYMS: + - PUE + - WUE + - BESS + - UPS + NOT_TECH_WORDS: + - community center + - recreation center + - civic center + - convention center + - shopping center + - medical center + - senior center + - visitor center + - town center + - detention center + - child care center + - call center + - distribution center + - service center + - data collection + - database + +collection_prompts: true + +extraction_system_prompt: |- + You are a legal scholar extracting structured data from local data + center siting ordinances. + + Extract ONLY the following features: + - setbacks from residential and protected receptors, property lines, + roads, transit facilities, parks and natural areas, other data + centers, and setbacks that apply to mechanical equipment under a + standard separate from the building + - minimum lot size, maximum lot coverage, minimum vegetative cover, + maximum height, maximum building size, data center share of project + floor area, and parking + - primary, special, accessory, and prohibited use districts, and + jurisdiction-wide prohibitions or moratoria + - noise limits, low-frequency and tonal noise limits, noise + attenuation standards for enclosures and barriers, vibration, light + illumination at a receptor, and thermal impact + - buffer and landscaping standards, equipment screening and + enclosure, and perimeter fencing + - backup generator operating restrictions, on-site power technology, + and decommissioning + - cooling technology, water consumption limits, and reclaimed water + requirements + + Extract only enacted local requirements that explicitly govern data + centers, or that govern a use the ordinance defines as a data center, + such as a data processing center or a computer server facility. The + source may be a general zoning code, unified development ordinance, + municipal code, or a standalone adopted ordinance rather than a + technology-specific title. + + Data center ordinances frequently regulate by facility size class. + Where a jurisdiction sets a different standard for each class, emit + one row per class and record the jurisdiction's own class name in + data_center_type rather than collapsing the tiers into one row or + mapping the name onto another jurisdiction's taxonomy. + + For a numeric feature, value is a single number: the most restrictive + value the ordinance sets for that feature and facility class, such as + the largest setback or the lowest noise limit. Do not combine several + numbers into one value. List every other value in summary with the + condition it applies under, such as the time period, receiving use, + zoning district, street class, or yard. Keep units to the unit alone. + + Set data_center_type to null when the requirement applies to data + centers generally. Generic labels such as Data Center, DATA CENTER, + and Data Processing Center are not specific facility types. Retain + specific class names such as Hyperscale Data Center or Micro Data + Center. For accessory data centers, use Accessory Data Center without + parenthetical conditions. Keep those conditions in summary and retain + the complete source wording in ordinance_text. + + Do not extract text focused only on office, warehouse, distribution, + telecommunications, or utility uses unless the same clause directly + applies to data centers. Do not extract from proposed, denied, or + superseded provisions. + + For noise, ignore complaint-response and monitoring protocol details + unless the clause states an explicit numeric limit. + + Follow schema feature definitions literally. Keep the jurisdiction's + own vocabulary exactly as written, except for the data_center_type + rules above. Write units with the schema's canonical unit names, such + as feet, acres, sq ft, percent, and dBA, even where the ordinance + abbreviates them. Do not convert values into a different unit. Do not + invent values. + + Write summary in your own words. Copy the supporting source passages + verbatim into ordinance_text, retaining the applicable facility class, + conditions, and exceptions. Use explanation to state how that evidence + supports the selected feature, facility class, value, and units. diff --git a/compass/pipeline/coordinator.py b/compass/pipeline/coordinator.py index 9319a294b..fad516e56 100644 --- a/compass/pipeline/coordinator.py +++ b/compass/pipeline/coordinator.py @@ -21,7 +21,7 @@ from compass.services.threaded import UsageUpdater from compass.services.threaded import GenericFuncRunner from compass.utilities.enums import COMPASSRunMode -from compass.utilities.jurisdictions import jurisdictions_from_df +from compass.utilities.jurisdictions import jurisdictions_from_df, fips_to_str from compass.utilities.logs import log_versions from compass.utilities.parsing import convert_paths_to_strings from compass.pipeline.collection.persistence import ( @@ -155,15 +155,18 @@ def _create( usage_tracker=usage_tracker, rate_tracker=self.runtime.rate_tracker, ) + source_key = fips_to_str(jurisdiction.code) return SingleJurisdictionRun( self.runtime, jurisdiction, extractor, usage_tracker=usage_tracker, - known_local_docs=self.runtime.known_local_docs.get( - jurisdiction.code + known_local_docs=_known_source_for_code( + self.runtime.known_local_docs, source_key + ), + known_doc_urls=_known_source_for_code( + self.runtime.known_doc_urls, source_key ), - known_doc_urls=self.runtime.known_doc_urls.get(jurisdiction.code), known_se_results=known_se_results, perform_se_search=self.runtime.request.perform_se_search, perform_website_search=( @@ -707,3 +710,12 @@ async def _compute_total_cost(rate_tracker): compute_total_cost_from_usage(total_usage), total_usage.get(rate_tracker.label), ) + + +def _known_source_for_code(sources, code): + """Prefer full codes, then accept keys without leading zeros""" + if code in sources: + return sources[code] + if code.isdecimal(): + return sources.get(code.lstrip("0") or "0") + return None diff --git a/compass/pipeline/runtime.py b/compass/pipeline/runtime.py index 312a6cec3..dec78075e 100644 --- a/compass/pipeline/runtime.py +++ b/compass/pipeline/runtime.py @@ -379,7 +379,7 @@ def _setup_folders( def _load_known_source(known_source): - """Load configured known sources as int-keyed dictionaries""" + """Load configured known sources with string jurisdiction codes""" known_source = known_source or {} if isinstance(known_source, str): known_source = load_config(known_source) diff --git a/compass/plugin/one_shot/components.py b/compass/plugin/one_shot/components.py index 84162441f..6918bb584 100644 --- a/compass/plugin/one_shot/components.py +++ b/compass/plugin/one_shot/components.py @@ -56,7 +56,11 @@ navigation links, cross-reference lists, or citation-only indexes as relevant \ on their own; mark them relevant only when they also include operative \ provisions, definitions, prohibitions, or other substantive regulatory text \ -that supports extraction. If relevant, summarize the specific matching \ +that supports extraction. The content must be about the specific technology \ +or use the schema targets; tangentially related material does not count, \ +such as a general definition or provision for some other use that merely \ +mentions a related activity in passing. \ +If relevant, summarize the specific matching \ content; if not, state why it does not meet the schema's requirements. \ Keep the response concise and consistent.\ """ @@ -516,8 +520,9 @@ async def parse(self, text, jurisdiction): ) data = extraction["outputs"] if not data: - logger.debug( - "LLM did not extract any relevant features from the text" + logger.info( + "LLM did not extract any relevant features from the text: %s", + extraction.get("extraction_notes") or "no explanation given", ) return None diff --git a/compass/plugin/ordinance.py b/compass/plugin/ordinance.py index 844c90a07..3accbeaae 100644 --- a/compass/plugin/ordinance.py +++ b/compass/plugin/ordinance.py @@ -60,6 +60,8 @@ "primary use districts", "special use districts", "accessory use districts", + # a definitions chapter defines terms but regulates nothing + "definition", } diff --git a/compass/plugin/post_processing.py b/compass/plugin/post_processing.py index 6d9bbc711..5e54d442d 100644 --- a/compass/plugin/post_processing.py +++ b/compass/plugin/post_processing.py @@ -1,6 +1,7 @@ """Post processing functions for one-shot plugins""" import inspect +import re from pathlib import Path @@ -30,6 +31,40 @@ def add_document_name(db): return db +def normalize_data_center_types(db): + """Leave generic types blank and shorten accessory type labels""" + db["data_center_type"] = db["data_center_type"].map( + _normalize_data_center_type + ) + return db + + +def _normalize_data_center_type(value): + if not isinstance(value, str): + return value + + value = " ".join(value.split()) + if value.casefold() in { + "", + "data center", + "data centers", + "datacenter", + "datacenters", + "all", + "all types", + "all data centers", + "all data center types", + }: + return None + + if re.fullmatch( + r"accessory data center(?:\s*\(.*\))?", value, re.IGNORECASE + ): + return "Accessory Data Center" + + return value + + POST_PROCESSING_REGISTRY = { name: func for name, func in globals().items() diff --git a/compass/utilities/costs.py b/compass/utilities/costs.py index 32e5c8c1c..384a63c68 100644 --- a/compass/utilities/costs.py +++ b/compass/utilities/costs.py @@ -25,6 +25,7 @@ "gpt-6-luna": {"prompt": 0.1, "response": 0.5}, "gpt-6-sol": {"prompt": 2, "response": 10}, "gpt-6-astra": {"prompt": 10, "response": 50}, + "gpt-6.1-sol": {"prompt": 2, "response": 10}, "text-embedding-ada-002": {"prompt": 0.10}, } """LLM Costs registry diff --git a/compass/utilities/jurisdictions.py b/compass/utilities/jurisdictions.py index 629e187df..d6b3043ec 100644 --- a/compass/utilities/jurisdictions.py +++ b/compass/utilities/jurisdictions.py @@ -326,8 +326,9 @@ def load_jurisdictions_from_fp(jurisdiction_fp): Parameters ---------- jurisdiction_fp : path-like - Path to csv file containing "County" and "State" columns that - define the jurisdictions for which info should be loaded. + Path to a CSV with "FIPS" or jurisdiction name columns. + When supplied, FIPS is the sole matching key. Codes must match + the catalog, including leading zeros. Returns ------- @@ -338,16 +339,19 @@ def load_jurisdictions_from_fp(jurisdiction_fp): Raises ------ COMPASSValueError - If the input file is missing required columns (``State`` or - ``Jurisdiction Type`` when subdivisions are provided). + If a file without FIPS is missing required name columns + (``State`` or ``Jurisdiction Type`` for subdivisions). Notes ----- Missing jurisdictions trigger warnings with a tabular summary. + A blank or unknown FIPS is skipped without falling back to names. """ jurisdictions = pd.read_csv(jurisdiction_fp, dtype=str).replace( {np.nan: None} ) + if "FIPS" in jurisdictions: + return _load_jurisdictions_by_fips(jurisdictions) jurisdictions = _validate_jurisdiction_input(jurisdictions) all_jurisdiction_info = load_all_jurisdiction_info() @@ -383,6 +387,28 @@ def load_jurisdictions_from_fp(jurisdiction_fp): return _format_jurisdiction_df_for_output(jurisdictions) +def _load_jurisdictions_by_fips(jurisdictions): + requested = jurisdictions[["FIPS"]].rename(columns={"FIPS": "FIPS_user"}) + requested["FIPS_user"] = requested["FIPS_user"].str.strip() + matched = requested.drop_duplicates().merge( + load_all_jurisdiction_info(), + left_on="FIPS_user", + right_on="FIPS", + how="left", + validate="one_to_one", + ) + matched = _filter_not_found_jurisdictions(matched, ["FIPS"]) + columns = [ + "County", + "State", + "Subdivision", + "Jurisdiction Type", + "FIPS", + "Website", + ] + return matched[columns].reset_index(drop=True) + + def jurisdictions_from_df(jurisdiction_info=None): """Convert rows DataFrame into Jurisdiction instances @@ -499,7 +525,10 @@ def _normalize_jurisdiction_name(name): def fips_to_str(value): - """[NOT PUBLIC API] Convert FIPS code to string""" + """[NOT PUBLIC API] Preserve strings when converting FIPS codes""" + if isinstance(value, str): + return value + with suppress(ValueError): value = int(value) return str(value) diff --git a/export_human_dc_ordinances.py b/export_human_dc_ordinances.py new file mode 100644 index 000000000..8c97db3ad --- /dev/null +++ b/export_human_dc_ordinances.py @@ -0,0 +1,212 @@ +import argparse +import csv +import re +from collections import Counter, defaultdict +from pathlib import Path +from urllib.parse import urlsplit + +import openpyxl + + +def main(): + parser = argparse.ArgumentParser() + base = Path(__file__).resolve().parent + parser.add_argument("--workbook", type=Path, default=base / "COMPASS DC Ordinance Research_2026.08.16.xlsx") + parser.add_argument("--reference", type=Path, default=base / "examples/one_shot_schema_extraction_dc/outputs/ordinances.csv") + parser.add_argument("--output", type=Path, default=base / "human_extracted_dc_ordinances.csv") + args = parser.parse_args() + workbook = openpyxl.load_workbook(args.workbook, data_only=True) + ordinances = sheet_rows(workbook["DC Ordinances"]) + definitions = sheet_rows(workbook["Definitions"]) + records = [ordinance_record(row) for row in ordinances] + records.extend(definition_record(row) for row in definitions) + assign_jurisdiction_metadata(records, base / "compass/data/conus_jurisdictions.csv") + with args.reference.open(encoding="utf-8-sig", newline="") as stream: + fields = [ + field for field in next(csv.reader(stream)) + if field in records[0] and field != "FIPS" + ] + fields.append("FIPS") + with args.output.open("w", encoding="utf-8-sig", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=fields) + writer.writeheader() + writer.writerows(records) + verify_export(args.output, fields, records, ordinances, definitions) + print(f"Wrote {len(records)} rows: {len(ordinances)} ordinance features and {len(definitions)} definitions.") + print("Data center types:", dict(Counter(record["data_center_type"] for record in records))) + for sheet in workbook["DC Ordinances"], workbook["Definitions"]: + report_link_mismatches(sheet) + + +def assign_jurisdiction_metadata(records, catalog_path): + """Resolve catalog types and FIPS using names and explicit name suffixes.""" + locations = defaultdict(list) + with catalog_path.open(encoding="utf-8-sig", newline="") as stream: + for row in csv.DictReader(stream): + locations[(row["State"].casefold(), row["Subdivision"].casefold())].append(row) + + for record in records: + state = record["state"].strip().casefold() + subdivision = (record["subdivision"] or "").strip().casefold() + county = (record["county"] or "").strip().casefold() + candidates = locations[(state, subdivision)] + if not candidates: + suffix = re.fullmatch(r"(.+) (city|town|township|village|borough)", subdivision) + if suffix: + candidates = [ + row for row in locations[(state, suffix[1])] + if row["Jurisdiction Type"].casefold() == suffix[2] + ] + if not subdivision: + candidates = [row for row in candidates if row["County"].casefold() == county] + elif county: + # Some catalog municipalities have no county recorded. + candidates = [row for row in candidates if row["County"].casefold() in {"", county}] + if len(candidates) != 1: + raise ValueError( + f"Expected one catalog match for {(state, county, subdivision)}, " + f"found {len(candidates)}" + ) + record["jurisdiction_type"] = candidates[0]["Jurisdiction Type"].title() + record["FIPS"] = candidates[0]["FIPS"] + + +def ordinance_record(row): + record = common_record(row) + record.update( + feature=normalize_feature(row["Feature"].value), + data_center_type=normalize_type(row["Data Center Type"].value), + value=row["Value"].value, + units="str" if row["Feature Data Type"].value == "Qualitative" else row["Units"].value, + summary=row["Summary"].value, + ) + return record + + +def definition_record(row): + record = common_record(row) + term = row["Defined Term"].value + record.update( + feature="definition", + data_center_type=normalize_type(term), + value=row["Definition (verbatim)"].value, + units="str", + summary=None, + explanation=f"Defined term: {term}.\n{record['explanation']}".rstrip(), + ) + return record + + +def common_record(row): + mapping = { + "county": "County", + "state": "State", + "subdivision": "City / Subdivision", + "jurisdiction_type": "Jurisdiction Type", + "section": "Section (Pin Cite)", + } + record = {key: row[column].value for key, column in mapping.items()} + raw_year = row["Ordinance Year"].value + year = int(re.match(r"^\s*(\d{4})\b", str(raw_year)).group(1)) + notes = row["Notes"].value or "" + # Keep adoption, repeal, and proposal annotations outside the numeric year. + if str(raw_year).strip() != str(year): + notes = f"{notes}\nOrdinance Year in workbook: {raw_year}".strip() + pdf = row.get("Direct PDF URL") + selected = pdf if pdf is not None and (pdf.value or pdf.hyperlink) else row["Source URL"] + record.update(year=year, explanation=notes, source=cell_url(selected)) + return record + + +def normalize_feature(value): + aliases = { + 'buffer / landscaping': 'buffer/landscaping', + 'equipment screening / enclosure': 'equipment screening/enclosure', + 'fencing / security screening': 'fencing/security screening', + 'heat / thermal impact': 'heat/thermal impact', + 'noise (low-frequency / tonal)': 'noise (low-frequency/tonal)', + 'noise mitigation / acoustic barriers': 'noise mitigation/acoustic barriers', + 'prohibitions / moratorium': 'prohibitions/moratorium', + 'setback - parks / natural areas': 'setback - parks/natural areas', + 'water reuse / reclaimed': 'water reuse/reclaimed', + } + return aliases.get(value.strip().casefold(), value) + + +def normalize_type(value): + if value is None: + return "" + term = value.strip() + # Parenthetical annotations describe the instrument, not a facility category. + if re.fullmatch(r"data centers?(?:\s*\(.*\))?", term, re.IGNORECASE): + return "" + if term.casefold() in {"all", "all types", "all data center types", "hyperscale / medium / micro"}: + return "" + aliases = { + "hyperscale": "Hyperscale Data Center", + "medium": "Medium Data Center", + "micro": "Micro Data Center", + "accessory": "Accessory Data Center", + "data center, hyperscale": "Hyperscale Data Center", + "data center, medium": "Medium Data Center", + "data center, micro": "Micro Data Center", + "data center, accessory": "Accessory Data Center", + "accessory use": "Accessory Data Center", + "data center as an accessory use": "Accessory Data Center", + "data center, accessory use": "Accessory Data Center", + "data center, ancillary": "Ancillary", + "data center, campus": "Campus", + "data center, major": "Major", + "data center, minor": "Minor", + } + return aliases.get(term.casefold(), term) + + +def verify_export(path, fields, records, ordinances, definitions): + with path.open(encoding="utf-8-sig", newline="") as stream: + reader = csv.DictReader(stream) + saved = list(reader) + assert reader.fieldnames == fields + assert len(saved) == len(ordinances) + len(definitions) + for actual, expected in zip(saved, records): + assert all(actual[key] == ("" if expected[key] is None else str(expected[key])) for key in fields) + assert type(expected["year"]) is int + assert re.fullmatch(r"\d{4}", actual["year"]) + assert actual["source"].startswith(("http://", "https://")) + assert not re.fullmatch(r"data center(?:\s*\(.*\))?", actual["data_center_type"], re.IGNORECASE) + for actual, row in zip(saved, ordinances): + if row["Feature Data Type"].value == "Qualitative": + assert actual["units"] == "str" + else: + units = row["Units"].value + assert actual["units"] == ("" if units is None else str(units)) + for actual, row in zip(saved[len(ordinances):], definitions): + assert actual["feature"] == "definition" and actual["units"] == "str" + assert actual["value"] == row["Definition (verbatim)"].value + print("Verified column order, all exported values, numeric years, qualitative units, and definition text.") + + +def sheet_rows(sheet): + headers = [cell.value for cell in sheet[1]] + return [ + dict(zip(headers, row)) + for row in sheet.iter_rows(min_row=2) + if any(cell.value is not None for cell in row) + ] + + +def cell_url(cell): + return cell.hyperlink.target if cell.hyperlink and cell.hyperlink.target else cell.value + + +def report_link_mismatches(sheet): + for row in sheet_rows(sheet): + cell = row["Source URL"] + visible = str(cell.value or "") + target = cell_url(cell) + if cell.hyperlink and visible.startswith("http") and urlsplit(visible).netloc != urlsplit(target).netloc: + print(f"Hyperlink host mismatch at {sheet.title}!{cell.coordinate}: {visible!r} -> {target!r}") + + +if __name__ == "__main__": + main() diff --git a/tests/python/integration/test_integrated_utilities_jurisdictions.py b/tests/python/integration/test_integrated_utilities_jurisdictions.py index 2c2300669..c06f337f5 100644 --- a/tests/python/integration/test_integrated_utilities_jurisdictions.py +++ b/tests/python/integration/test_integrated_utilities_jurisdictions.py @@ -50,6 +50,29 @@ def test_load_all_jurisdictions(): assert "Rhode Island" in set(jurisdiction_info["State"]) +def test_fips_matching_with_missing_catalog_county(tmp_path): + """Match actual catalog codes across county omissions and place types.""" + path = tmp_path / "jurisdictions.csv" + pd.DataFrame( + { + "FIPS": ["0107000", "1723256", "4204183800", "5183136"], + "County": ["Jefferson", "Cook", "Cumberland", "Fauquier"], + } + ).to_csv(path, index=False) + + result = load_jurisdictions_from_fp(path) + + assert result["FIPS"].tolist() == [ + "0107000", "1723256", "4204183800", "5183136" + ] + assert result["Jurisdiction Type"].tolist() == [ + "city", "village", "township", "town" + ] + assert result["County"].tolist() == [ + "Jefferson", None, "Cumberland", None + ] + + def test_jurisdiction_websites(): """Test the `jurisdiction_websites` function""" diff --git a/tests/python/unit/extraction/test_extraction_data_centers_schema.py b/tests/python/unit/extraction/test_extraction_data_centers_schema.py new file mode 100644 index 000000000..b36d99542 --- /dev/null +++ b/tests/python/unit/extraction/test_extraction_data_centers_schema.py @@ -0,0 +1,79 @@ +"""Checks for the data center extraction schema contract""" + +import json +from pathlib import Path + +import compass + + +SCHEMA_FP = ( + Path(compass.__file__).parent + / "extraction" + / "data_centers" + / "data_centers_schema.json" +) + + +def test_schema_asks_for_extraction_notes(): + """Every extraction explains itself, even when it finds nothing""" + schema = json.loads(SCHEMA_FP.read_text(encoding="utf-8")) + assert schema["required"] == ["outputs", "extraction_notes"] + notes = schema["properties"]["extraction_notes"] + assert notes["type"] == "string" + assert "outputs is empty" in notes["description"] + assert schema["additionalProperties"] is False + + +def test_schema_extracts_definitions_of_data_center_terms(): + """Definitions are a feature, described and marked qualitative""" + schema = json.loads(SCHEMA_FP.read_text(encoding="utf-8")) + features = schema["properties"]["outputs"]["items"]["properties"][ + "feature" + ]["enum"] + assert "definition" in features + described = { + name for group in schema["$definitions"].values() for name in group + } + assert described == set(features) + assert "definition" in schema["$qualitative_features"] + assert any( + f"{len(features)} allowed feature IDs" in item + for item in schema["$qa_checklist"] + ) + + +def test_numeric_features_ask_for_most_restrictive_value(): + """Numeric rows hold one value; the rest belong in the summary""" + schema = json.loads(SCHEMA_FP.read_text(encoding="utf-8")) + assert "compound" not in json.dumps(schema).casefold() + assert "numeric_prioritization" in schema["$instructions"] + + qualitative = set(schema["$qualitative_features"]) + definitions = { + name: spec["description"] + for group in schema["$definitions"].values() + for name, spec in group.items() + } + # a definition row's value is the definition text, not a requirement + definitions.pop("definition") + for name, description in definitions.items(): + value_rule = description.split("VALUE:")[1].split("UNITS:")[0] + if name in qualitative: + assert value_rule.strip().startswith("null"), name + elif not name.endswith("districts"): + assert "largest" in value_rule or "lowest" in value_rule, name + + +def test_units_use_canonical_vocabulary_without_detail(): + """Units name a canonical unit; any other detail goes in summary""" + schema = json.loads(SCHEMA_FP.read_text(encoding="utf-8")) + vocabulary = schema["$instructions"]["units"] + assert any("canonical vocabulary" in item for item in vocabulary) + + for group in schema["$definitions"].values(): + for name, spec in group.items(): + units_rule = spec["description"].split("UNITS:")[1] + units_rule = units_rule.split("SUMMARY:")[0].split("IGNORE:")[0] + assert "exactly as written" not in units_rule, name + if name.endswith("districts"): + assert units_rule.strip() == "null.", name diff --git a/tests/python/unit/pipeline/test_pipeline_orchestration.py b/tests/python/unit/pipeline/test_pipeline_orchestration.py index 6f2a87106..d5a56d54e 100644 --- a/tests/python/unit/pipeline/test_pipeline_orchestration.py +++ b/tests/python/unit/pipeline/test_pipeline_orchestration.py @@ -1,9 +1,11 @@ """Tests for compass.pipeline orchestration""" +import json import logging from itertools import product from pathlib import Path from types import SimpleNamespace +from unittest.mock import Mock import pandas as pd import pytest @@ -83,6 +85,31 @@ async def run(self, jurisdictions_df): return DummyWorkflow +@pytest.mark.parametrize("from_file", [False, True]) +def test_known_sources_preserve_fips(tmp_path, from_file): + """Keep leading zeros in local-document and URL lookup keys""" + known_local_docs = {"0648914": [{"source_fp": "ordinance.pdf"}]} + known_doc_urls = {"0648914": ["https://example.com/ordinance.pdf"]} + if from_file: + local_fp = tmp_path / "local.json" + local_fp.write_text(json.dumps(known_local_docs), encoding="utf-8") + urls_fp = tmp_path / "urls.json" + urls_fp.write_text(json.dumps(known_doc_urls), encoding="utf-8") + + request = CollectionRequest( + out_dir=tmp_path / "outputs", + tech="solar", + jurisdiction_fp=tmp_path / "jurisdictions.csv", + known_local_docs=str(local_fp) if from_file else known_local_docs, + known_doc_urls=str(urls_fp) if from_file else known_doc_urls, + ) + + runtime = PipelineRuntime(request) + + assert runtime.known_local_docs == known_local_docs + assert runtime.known_doc_urls == known_doc_urls + + def test_known_local_docs_missing_file(tmp_path): """Raise when known_local_docs points to missing config""" missing_fp = tmp_path / "does_not_exist.json" @@ -100,6 +127,40 @@ def test_known_local_docs_missing_file(tmp_path): PipelineRuntime(request) +@pytest.mark.parametrize("use_canonical_key", [False, True]) +def test_known_sources_match_fips_with_leading_zero( + monkeypatch, use_canonical_key +): + """Canonical FIPS codes still find numerically keyed known sources.""" + runtime = SimpleNamespace( + extractor_class=Mock(), + models={}, + rate_tracker=None, + search_params=SimpleNamespace(num_search_results_to_crawl=0), + known_local_docs={"8041": ["ordinance.pdf"]}, + known_doc_urls={"8041": ["https://example.com/ordinance.pdf"]}, + request=SimpleNamespace( + perform_se_search=True, perform_website_search=False + ), + ) + if use_canonical_key: + runtime.known_local_docs["08041"] = ["canonical.pdf"] + runtime.known_doc_urls["08041"] = ["https://example.com/canonical.pdf"] + create_run = Mock() + monkeypatch.setattr( + coordinator_module, "SingleJurisdictionRun", create_run + ) + workflow = coordinator_module.COMPASSFullProcessing(runtime) + + workflow._create(SimpleNamespace(code="08041")) + + filename = "canonical.pdf" if use_canonical_key else "ordinance.pdf" + assert create_run.call_args.kwargs["known_local_docs"] == [filename] + assert create_run.call_args.kwargs["known_doc_urls"] == [ + f"https://example.com/{filename}" + ] + + def test_known_local_docs_logs_missing_file(tmp_path, testing_log_file): """Log missing known_local_docs config to error file""" diff --git a/tests/python/unit/plugin/test_plugin_ordinances.py b/tests/python/unit/plugin/test_plugin_ordinances.py index 83d8c287c..b392c0afe 100644 --- a/tests/python/unit/plugin/test_plugin_ordinances.py +++ b/tests/python/unit/plugin/test_plugin_ordinances.py @@ -937,3 +937,18 @@ async def test_single_doc_extraction_runs_out_of_documents(tmp_path): "extract.doc.no_data", "no documents left", ) + + +def test_definitions_alone_do_not_make_an_ordinance(): + """A definitions chapter regulates nothing, so extraction moves on""" + data = pd.DataFrame( + { + "feature": ["definition", "definition", "noise"], + "value": ["A facility housing servers", "A large facility", None], + "summary": [None, None, None], + } + ) + count = OrdinanceExtractionPlugin.get_structured_data_row_count + assert count(data) == 0 + data.loc[2, "value"] = "55 dBA" + assert count(data) == 1 diff --git a/tests/python/unit/plugin/test_plugin_post_processing.py b/tests/python/unit/plugin/test_plugin_post_processing.py new file mode 100644 index 000000000..b19d27994 --- /dev/null +++ b/tests/python/unit/plugin/test_plugin_post_processing.py @@ -0,0 +1,99 @@ +"""COMPASS plugin post-processing tests""" + +from pathlib import Path + +import pandas as pd +import pytest + +from compass.extraction.data_centers import COMPASSDataCentersExtractor +from compass.utilities.jurisdictions import Jurisdiction +from compass.plugin.post_processing import normalize_data_center_types + + +@pytest.mark.parametrize( + ("label", "expected"), + [ + (None, None), + ("", None), + ("Data Center", None), + ("DATA Center", None), + (" DATA CENTER ", None), + ("Data Centers", None), + ("All data center types", None), + ("Accessory Data Center ", "Accessory Data Center"), + ( + ( + "Accessory Data Center " + "(as Accessory to an Office or Industrial Use)" + ), + "Accessory Data Center", + ), + ("accessory data center (office use)", "Accessory Data Center"), + ("Hyperscale Data Center", "Hyperscale Data Center"), + ("Micro Data Center", "Micro Data Center"), + ("Data Center, Medium", "Data Center, Medium"), + ("Data Center (Micro)", "Data Center (Micro)"), + ], +) +def test_normalize_data_center_types(label, expected): + """Normalize generic and accessory labels without changing evidence""" + db = pd.DataFrame( + [{"data_center_type": label, "ordinance_text": "Original wording"}] + ) + + out = normalize_data_center_types(db) + + assert out.loc[0, "data_center_type"] == expected + assert out.loc[0, "ordinance_text"] == "Original wording" + + +def test_data_center_export_preserves_codes_and_normalizes_types(tmp_path): + """Apply type rules and keep FIPS codes in the combined export""" + jurisdiction = Jurisdiction( + "city", "Arizona", subdivision_name="Mesa", code="0446000" + ) + source_dir = tmp_path / "jurisdiction_dbs" + source_dir.mkdir() + source_fp = source_dir / f"{jurisdiction.full_name} Ordinances.csv" + labels = [ + "DATA CENTER", + "Accessory Data Center (as Accessory to an Office or Industrial Use)", + "Hyperscale Data Center", + ] + pd.DataFrame( + [ + { + "feature": "maximum height", + "data_center_type": label, + "value": 40, + "units": "feet", + "summary": "Maximum height is 40 feet.", + "ordinance_text": "Original wording", + "explanation": "The text states a height limit.", + "quantitative": True, + "year": 2026, + "source": "https://example.com/ordinance.pdf", + } + for label in labels + ] + ).to_csv(source_fp, index=False) + + COMPASSDataCentersExtractor.save_structured_data( + [{"ord_db_fp": source_fp, "jurisdiction": jurisdiction}], + tmp_path, + ) + + out = pd.read_csv( + tmp_path / "ordinances.csv", dtype=str, keep_default_na=False + ) + assert out["FIPS"].tolist() == ["0446000"] * len(labels) + assert out["data_center_type"].tolist() == [ + "", + "Accessory Data Center", + "Hyperscale Data Center", + ] + assert out["ordinance_text"].tolist() == ["Original wording"] * len(labels) + + +if __name__ == "__main__": + pytest.main(["-q", "--show-capture=all", Path(__file__), "-rapP"]) diff --git a/tests/python/unit/utilities/test_utilities_jurisdictions.py b/tests/python/unit/utilities/test_utilities_jurisdictions.py index 78f952273..4fb73a811 100644 --- a/tests/python/unit/utilities/test_utilities_jurisdictions.py +++ b/tests/python/unit/utilities/test_utilities_jurisdictions.py @@ -7,6 +7,7 @@ import numpy as np from compass.utilities.jurisdictions import ( + fips_to_str, load_jurisdictions_from_fp, load_jurisdictions_from_subdivision_names, jurisdiction_websites, @@ -123,6 +124,102 @@ def test_load_jurisdictions_from_fp(tmp_path): assert {type(val) for val in jurisdictions["FIPS"]} == {str} +def test_load_jurisdictions_by_fips_ignores_names(tmp_path): + """FIPS overrides conflicting names and deduplicates requests.""" + path = tmp_path / "jurisdictions.csv" + pd.DataFrame( + { + "FIPS": ["08041", " 08041 ", "49003"], + "State": ["Wrong state"] * 3, + "County": ["Wrong county"] * 3, + } + ).to_csv(path, index=False) + + result = load_jurisdictions_from_fp(path) + + assert result["FIPS"].tolist() == ["08041", "49003"] + assert result["State"].tolist() == ["Colorado", "Utah"] + assert result["County"].tolist() == ["El Paso", "Box Elder"] + + +def test_load_jurisdictions_from_fips_only(tmp_path): + """A FIPS-only file preserves leading zeros without requiring names.""" + path = tmp_path / "jurisdictions.csv" + pd.DataFrame({"FIPS": ["08041"]}).to_csv(path, index=False) + + result = load_jurisdictions_from_fp(path) + + assert result["FIPS"].tolist() == ["08041"] + assert result["Jurisdiction Type"].tolist() == ["county"] + + +def test_load_jurisdictions_invalid_fips_do_not_fall_back_to_names(tmp_path): + """Unknown and missing codes cannot select a different named location.""" + path = tmp_path / "jurisdictions.csv" + pd.DataFrame( + { + "FIPS": ["9999999999", None, "08041"], + "State": ["Colorado"] * 3, + "County": ["El Paso"] * 3, + } + ).to_csv(path, index=False) + + with pytest.warns(COMPASSWarning, match="9999999999"): + result = load_jurisdictions_from_fp(path) + + assert result["FIPS"].tolist() == ["08041"] + + +@pytest.mark.parametrize( + ("state", "county", "subdivision", "jurisdiction_type", "fips"), + [ + ("California", None, "Monterey Park", "city", "0648914"), + ("Colorado", "Adams", None, "county", "08001"), + ("California", None, None, "state", "06000"), + ("Alabama", "Autauga", "Marbury", "city", "0100192106"), + ], +) +def test_load_jurisdictions_from_fp_preserves_fips( + tmp_path, state, county, subdivision, jurisdiction_type, fips +): + """Preserve canonical codes through CSV lookup and object creation""" + jurisdiction_fp = tmp_path / "jurisdictions.csv" + pd.DataFrame( + { + "State": [state], + "County": [county], + "Subdivision": [subdivision], + "Jurisdiction Type": [jurisdiction_type], + } + ).to_csv(jurisdiction_fp, index=False) + + jurisdictions = load_jurisdictions_from_fp(jurisdiction_fp) + + assert jurisdictions["FIPS"].tolist() == [fips] + (jurisdiction,) = jurisdictions_from_df(jurisdictions) + assert jurisdiction.code == fips + assert jurisdiction.website_url == jurisdiction_websites()[fips] + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("0648914", "0648914"), + ("08001", "08001"), + ("06", "06"), + ("0100192106", "0100192106"), + ("district-01", "district-01"), + (18031, "18031"), + (18031.0, "18031"), + (np.int64(18031), "18031"), + (np.float64(18031), "18031"), + ], +) +def test_fips_to_str(value, expected): + """Preserve string identifiers and support numeric codes""" + assert fips_to_str(value) == expected + + def test_load_jurisdictions_from_fp_bad_input(tmp_path): """Test `load_jurisdictions_from_fp` function""" From 32bbb2cf4057fd6ec2bd72d75b77d35f88b3461a Mon Sep 17 00:00:00 2001 From: Rajendra Adhikari Date: Mon, 9 Nov 2026 17:40:45 -0600 Subject: [PATCH 02/13] Limit data center types to a fixed list --- collapse_human_dc_ordinances.py | 62 +++++++++++++++++++ .../data_centers/data_centers_schema.json | 33 +++++++--- .../data_centers/plugin_config.yaml | 25 ++++---- compass/plugin/post_processing.py | 35 ----------- .../test_extraction_data_centers_schema.py | 22 +++++++ .../plugin/test_plugin_post_processing.py | 48 +------------- 6 files changed, 121 insertions(+), 104 deletions(-) diff --git a/collapse_human_dc_ordinances.py b/collapse_human_dc_ordinances.py index ed028c044..e759a0469 100644 --- a/collapse_human_dc_ordinances.py +++ b/collapse_human_dc_ordinances.py @@ -110,6 +110,12 @@ def main(): before = [row.copy() for row in result] plain = use_plain_units(result) validate_units(before, result) + before = [row.copy() for row in result] + standard = use_standard_types(result) + validate_types(before, result) + shared = sum(Counter(row_key(row) for row in result).values()) - len( + {row_key(row) for row in result} + ) with output.open("w", encoding="utf-8-sig", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=columns) writer.writeheader() @@ -127,6 +133,10 @@ def main(): print(f"{qualitative} qualitative rows now carry their summary as value.") print(f"{filled} rows had an empty value; copied their summary into value.") print(f"{plain} rows had unit detail moved into their summary.") + print( + f"{standard} rows had a local class name mapped to a standard type; " + f"{shared} rows now repeat another row's key." + ) print( f"Validated unique keys, unchanged singleton rows, integer years, and CSV round-trip: {output}" ) @@ -372,6 +382,58 @@ def merge_metadata(rows): return row +# The fixed list the extraction schema allows for data_center_type, keyed +# by the class names the workbook uses. An empty type is a generic one. +STANDARD_TYPES = { + "": "", + "data processing center": "", + "data processing facility": "", + "hyperscale data center": "Hyperscale Data Center", + "large data center": "Large Data Center", + "large-scale data center": "Large Data Center", + "major": "Large Data Center", + "medium data center": "Medium Data Center", + "small urban data center": "Small Data Center", + "minor": "Small Data Center", + "micro data center": "Micro Data Center", + "accessory data center": "Accessory Data Center", + "ancillary": "Accessory Data Center", + "associated data room or server room": "Accessory Data Center", + "campus": "Data Center Campus", + "large quantity water user": "Other", +} + + +def use_standard_types(rows): + """Map each local class name to a standard type, keeping it in summary. + + Runs last: the steps above tell rows apart by their local class name. + """ + changed = 0 + for row in rows: + local = row["data_center_type"].strip() + standard = STANDARD_TYPES[local.casefold()] + if standard == local: + continue + row["data_center_type"] = standard + row["summary"] = f"{local}: {row['summary']}" + changed += 1 + return changed + + +def validate_types(before, after): + """Only the type and summary changed, and every type is standard.""" + allowed = set(STANDARD_TYPES.values()) + for old, new in zip(before, after, strict=True): + assert new["data_center_type"] in allowed + if new == old: + continue + local = old["data_center_type"].strip() + assert new["summary"] == f"{local}: {old['summary']}" + rewritten = {"data_center_type", "summary"} + assert all(new[k] == v for k, v in old.items() if k not in rewritten) + + def validate(result, groups): counts = Counter(row_key(row) for row in result) assert all(count == 1 for count in counts.values()) diff --git a/compass/extraction/data_centers/data_centers_schema.json b/compass/extraction/data_centers/data_centers_schema.json index 7f8622820..3a40a513f 100644 --- a/compass/extraction/data_centers/data_centers_schema.json +++ b/compass/extraction/data_centers/data_centers_schema.json @@ -11,7 +11,7 @@ "properties": { "outputs": { "type": "array", - "description": "Sparse long-form extraction table. Emit a row only for a feature with an explicit, enacted requirement. Emit at most one row per (feature, data center class) pair: where an ordinance sets a different standard for each size class of data center, emit one row per class rather than collapsing them.", + "description": "Sparse long-form extraction table. Emit a row only for a feature with an explicit, enacted requirement. Emit at most one row per feature and class of data center the ordinance names: where an ordinance sets a different standard for each class, emit one row per class rather than collapsing them.", "items": { "type": "object", "required": [ @@ -72,7 +72,18 @@ "string", "null" ], - "description": "The specific facility class governed by the provision, such as 'Hyperscale Data Center', 'Micro Data Center', or a jurisdiction-specific size class. Use null when the requirement applies to data centers generally, including generic terms such as 'Data Center', 'DATA CENTER', or 'Data Processing Center'. For accessory data centers, use 'Accessory Data Center' without parenthetical conditions. Keep conditions in summary and preserve the complete source wording in ordinance_text." + "description": "The standard facility class the provision governs. Must be one of the enumerated values or null; do not write the jurisdiction's own class name here. Choose by the rules in $instructions.size_tiering. Use null when the requirement applies to data centers generally. The jurisdiction's own class name and any conditions belong in summary, and the complete source wording in ordinance_text.", + "enum": [ + "Hyperscale Data Center", + "Large Data Center", + "Medium Data Center", + "Small Data Center", + "Micro Data Center", + "Accessory Data Center", + "Data Center Campus", + "Other", + null + ] }, "value": { "description": "The extracted requirement. For a numeric feature, a single number: the most restrictive value the ordinance sets for this feature and data center class, chosen by the rules in $instructions.numeric_prioritization. Never combine several numbers into one value; the other values belong in summary. Array of district names/codes for district features, boolean true where a requirement exists but is not numeric, or null where the feature is qualitative and the substance lives in the summary.", @@ -116,7 +127,7 @@ "string", "null" ], - "description": "A short summary of the requirement in your own words, retaining all specifics and conditions, such as an exception for a power outage, a standard that steps up next to a residential district, or a sunset date. Where the ordinance sets several values for this feature, state which one value holds and list every other value with the condition it applies under, such as the time period, receiving use, zoning district, street class, or yard. The verbatim excerpt belongs in ordinance_text. For qualitative features, state the requirement in full." + "description": "A short summary of the requirement in your own words. Where data_center_type is not null, start with the jurisdiction's own name for the class, exactly as the ordinance writes it. Retain all specifics and conditions, such as an exception for a power outage, a standard that steps up next to a residential district, or a sunset date. Where the ordinance sets several values for this feature, state which one value holds and list every other value with the condition it applies under, such as the time period, receiving use, zoning district, street class, or yard. The verbatim excerpt belongs in ordinance_text. For qualitative features, state the requirement in full." }, "ordinance_text": { "type": "string", @@ -253,7 +264,7 @@ }, "terms": { "definition": { - "description": "The jurisdiction's definition of a data center term, such as 'Data Center', 'Data Processing Center', or a size class like 'Hyperscale Data Center'. Emit one row per defined term, with data_center_type set to the defined class name, or null for the generic data center term. VALUE: the definition text, verbatim. UNITS: null. SECTION: where the term is defined. IGNORE: definitions of other uses that merely mention data processing or computing in passing, such as a business office that lists data processing among its activities." + "description": "The jurisdiction's definition of a data center term, such as 'Data Center', 'Data Processing Center', or a size class like 'Hyperscale Data Center'. Emit one row per defined term, with data_center_type set to the standard class the term maps to, or null for the generic data center term. SUMMARY: start with the defined term as written. VALUE: the definition text, verbatim. UNITS: null. SECTION: where the term is defined. IGNORE: definitions of other uses that merely mention data processing or computing in passing, such as a business office that lists data processing among its activities." } } }, @@ -262,14 +273,16 @@ "Extract only enacted local requirements that explicitly govern data centers, or that govern a use the ordinance defines as a data center, such as a data processing center or a computer server facility.", "Also record each definition of a data center term as a 'definition' row, even though a definition is not itself a requirement.", "Write summary in your own words. Put verbatim source passages in ordinance_text and explain how they support the row in explanation.", - "Never convert units. Preserve the jurisdiction's vocabulary except for the data_center_type rules and the canonical unit names. Keep ordinance_text verbatim.", + "Never convert units. Preserve the jurisdiction's vocabulary except for the standard data_center_type values and the canonical unit names. Keep ordinance_text verbatim.", "Do not infer a requirement from related context, and do not emit a row for a feature the source does not address." ], "size_tiering": [ - "Data center ordinances frequently set different standards for different size classes of facility. When they do, emit one row per class and record the jurisdiction's own class name in data_center_type.", - "Where one standard applies to data centers generally, emit a single row with data_center_type set to null. Generic labels such as 'Data Center' or 'DATA CENTER' must not appear as a specific facility class.", - "Do not merge tiered standards into a single row, and do not map a local class name onto another jurisdiction's taxonomy.", - "For accessory data centers, use 'Accessory Data Center' without parenthetical conditions. Retain those conditions in summary and preserve them verbatim in ordinance_text." + "Data center ordinances frequently set different standards for different classes of facility. When they do, emit one row per class the ordinance names, and do not merge tiered standards into a single row.", + "data_center_type holds a standard class, never the jurisdiction's wording. Map the local class by its name: 'Hyperscale' -> 'Hyperscale Data Center'; 'Large', 'Large-Scale', or 'Major' -> 'Large Data Center'; 'Medium' or 'Mid-Size' -> 'Medium Data Center'; 'Small' or 'Minor' -> 'Small Data Center'; 'Micro' or 'Edge' -> 'Micro Data Center'; accessory, ancillary, or incidental to another principal use -> 'Accessory Data Center'; 'Campus' or a multi-building data center development -> 'Data Center Campus'.", + "Map by the name the ordinance gives the class, not by the size thresholds behind it: one jurisdiction's 'Large' may be smaller than another's 'Medium'.", + "Use 'Other' for a specific class that fits none of the standard values, such as a class defined by whether the site is part of a unified development plan.", + "Use null where one standard applies to data centers generally. Generic terms such as 'Data Center', 'DATA CENTER', 'Data Processing Center', or 'Data Processing Facility' are not a specific class.", + "Two local classes may share a standard value. Keep them as separate rows, and start each summary with the jurisdiction's own class name so the rows can be told apart." ], "setbacks": [ "Setbacks are minimum separation distances. A setback row carries a single numeric distance and its unit.", @@ -301,7 +314,7 @@ "$qa_checklist": [ "All output rows use only the 35 allowed feature IDs.", "Every row includes a non-empty ordinance_text excerpt and an explanation linking that evidence to the feature and facility class.", - "Generic requirements have null data_center_type. Specific facility classes retain their names, tiered standards have one row per class, and accessory types use 'Accessory Data Center' without parenthetical conditions.", + "Every data_center_type is one of the standard values or null. Generic requirements have null, tiered standards have one row per class, and each row with a non-null type names the jurisdiction's own class at the start of summary.", "Numeric setbacks, lot standards, and building size rows carry numeric values and non-null units.", "Numeric rows carry a single number, the most restrictive value, with a plain unit, and summary lists the other values with their conditions.", "Units use only the canonical vocabulary in $instructions.units and carry no measurement detail.", diff --git a/compass/extraction/data_centers/plugin_config.yaml b/compass/extraction/data_centers/plugin_config.yaml index 69e3f9901..9f0832992 100644 --- a/compass/extraction/data_centers/plugin_config.yaml +++ b/compass/extraction/data_centers/plugin_config.yaml @@ -4,9 +4,6 @@ data_type_short_desc: data center ordinance cache_llm_generated_content: true -post_processing_steps: - - normalize_data_center_types - query_templates: - "{jurisdiction} data center ordinance" # Recently adopted ordinances are often published only as a PDF on the @@ -124,9 +121,7 @@ extraction_system_prompt: |- Data center ordinances frequently regulate by facility size class. Where a jurisdiction sets a different standard for each class, emit - one row per class and record the jurisdiction's own class name in - data_center_type rather than collapsing the tiers into one row or - mapping the name onto another jurisdiction's taxonomy. + one row per class rather than collapsing the tiers into one row. For a numeric feature, value is a single number: the most restrictive value the ordinance sets for that feature and facility class, such as @@ -135,13 +130,15 @@ extraction_system_prompt: |- condition it applies under, such as the time period, receiving use, zoning district, street class, or yard. Keep units to the unit alone. - Set data_center_type to null when the requirement applies to data - centers generally. Generic labels such as Data Center, DATA CENTER, - and Data Processing Center are not specific facility types. Retain - specific class names such as Hyperscale Data Center or Micro Data - Center. For accessory data centers, use Accessory Data Center without - parenthetical conditions. Keep those conditions in summary and retain - the complete source wording in ordinance_text. + data_center_type takes only the schema's standard values. Map the + jurisdiction's class onto the closest one by its name, such as Large + Data Center for a Large-Scale or Major class, and use Other for a + specific class that fits none of them. Set it to null when the + requirement applies to data centers generally: generic labels such as + Data Center, DATA CENTER, and Data Processing Center are not specific + classes. Start summary with the jurisdiction's own class name whenever + data_center_type is not null, and retain the complete source wording + in ordinance_text. Do not extract text focused only on office, warehouse, distribution, telecommunications, or utility uses unless the same clause directly @@ -153,7 +150,7 @@ extraction_system_prompt: |- Follow schema feature definitions literally. Keep the jurisdiction's own vocabulary exactly as written, except for the data_center_type - rules above. Write units with the schema's canonical unit names, such + values above. Write units with the schema's canonical unit names, such as feet, acres, sq ft, percent, and dBA, even where the ordinance abbreviates them. Do not convert values into a different unit. Do not invent values. diff --git a/compass/plugin/post_processing.py b/compass/plugin/post_processing.py index 5e54d442d..6d9bbc711 100644 --- a/compass/plugin/post_processing.py +++ b/compass/plugin/post_processing.py @@ -1,7 +1,6 @@ """Post processing functions for one-shot plugins""" import inspect -import re from pathlib import Path @@ -31,40 +30,6 @@ def add_document_name(db): return db -def normalize_data_center_types(db): - """Leave generic types blank and shorten accessory type labels""" - db["data_center_type"] = db["data_center_type"].map( - _normalize_data_center_type - ) - return db - - -def _normalize_data_center_type(value): - if not isinstance(value, str): - return value - - value = " ".join(value.split()) - if value.casefold() in { - "", - "data center", - "data centers", - "datacenter", - "datacenters", - "all", - "all types", - "all data centers", - "all data center types", - }: - return None - - if re.fullmatch( - r"accessory data center(?:\s*\(.*\))?", value, re.IGNORECASE - ): - return "Accessory Data Center" - - return value - - POST_PROCESSING_REGISTRY = { name: func for name, func in globals().items() diff --git a/tests/python/unit/extraction/test_extraction_data_centers_schema.py b/tests/python/unit/extraction/test_extraction_data_centers_schema.py index b36d99542..fbb63d2a7 100644 --- a/tests/python/unit/extraction/test_extraction_data_centers_schema.py +++ b/tests/python/unit/extraction/test_extraction_data_centers_schema.py @@ -42,6 +42,28 @@ def test_schema_extracts_definitions_of_data_center_terms(): ) +def test_data_center_type_is_a_fixed_list(): + """Types come from a closed list, so similar names cannot diverge""" + schema = json.loads(SCHEMA_FP.read_text(encoding="utf-8")) + spec = schema["properties"]["outputs"]["items"]["properties"][ + "data_center_type" + ] + assert spec["enum"] == [ + "Hyperscale Data Center", + "Large Data Center", + "Medium Data Center", + "Small Data Center", + "Micro Data Center", + "Accessory Data Center", + "Data Center Campus", + "Other", + None, + ] + rules = " ".join(schema["$instructions"]["size_tiering"]) + for name in spec["enum"][:-1]: + assert f"'{name}'" in rules, name + + def test_numeric_features_ask_for_most_restrictive_value(): """Numeric rows hold one value; the rest belong in the summary""" schema = json.loads(SCHEMA_FP.read_text(encoding="utf-8")) diff --git a/tests/python/unit/plugin/test_plugin_post_processing.py b/tests/python/unit/plugin/test_plugin_post_processing.py index b19d27994..1fba6f3d5 100644 --- a/tests/python/unit/plugin/test_plugin_post_processing.py +++ b/tests/python/unit/plugin/test_plugin_post_processing.py @@ -7,59 +7,17 @@ from compass.extraction.data_centers import COMPASSDataCentersExtractor from compass.utilities.jurisdictions import Jurisdiction -from compass.plugin.post_processing import normalize_data_center_types -@pytest.mark.parametrize( - ("label", "expected"), - [ - (None, None), - ("", None), - ("Data Center", None), - ("DATA Center", None), - (" DATA CENTER ", None), - ("Data Centers", None), - ("All data center types", None), - ("Accessory Data Center ", "Accessory Data Center"), - ( - ( - "Accessory Data Center " - "(as Accessory to an Office or Industrial Use)" - ), - "Accessory Data Center", - ), - ("accessory data center (office use)", "Accessory Data Center"), - ("Hyperscale Data Center", "Hyperscale Data Center"), - ("Micro Data Center", "Micro Data Center"), - ("Data Center, Medium", "Data Center, Medium"), - ("Data Center (Micro)", "Data Center (Micro)"), - ], -) -def test_normalize_data_center_types(label, expected): - """Normalize generic and accessory labels without changing evidence""" - db = pd.DataFrame( - [{"data_center_type": label, "ordinance_text": "Original wording"}] - ) - - out = normalize_data_center_types(db) - - assert out.loc[0, "data_center_type"] == expected - assert out.loc[0, "ordinance_text"] == "Original wording" - - -def test_data_center_export_preserves_codes_and_normalizes_types(tmp_path): - """Apply type rules and keep FIPS codes in the combined export""" +def test_data_center_export_preserves_codes_and_types(tmp_path): + """FIPS codes and data center types survive the combined export""" jurisdiction = Jurisdiction( "city", "Arizona", subdivision_name="Mesa", code="0446000" ) source_dir = tmp_path / "jurisdiction_dbs" source_dir.mkdir() source_fp = source_dir / f"{jurisdiction.full_name} Ordinances.csv" - labels = [ - "DATA CENTER", - "Accessory Data Center (as Accessory to an Office or Industrial Use)", - "Hyperscale Data Center", - ] + labels = [None, "Accessory Data Center", "Hyperscale Data Center"] pd.DataFrame( [ { From b715a60c5b5dc3c8d14165b3c8d79dd2aa607a7c Mon Sep 17 00:00:00 2001 From: Rajendra Adhikari Date: Mon, 9 Nov 2026 17:49:31 -0600 Subject: [PATCH 03/13] Keep feature and data center type unique per jurisdiction --- collapse_human_dc_ordinances.py | 11 +++++------ .../data_centers/data_centers_schema.json | 16 ++++++++-------- .../extraction/data_centers/plugin_config.yaml | 4 +++- .../test_extraction_data_centers_schema.py | 2 ++ 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/collapse_human_dc_ordinances.py b/collapse_human_dc_ordinances.py index e759a0469..d6f5b57b5 100644 --- a/collapse_human_dc_ordinances.py +++ b/collapse_human_dc_ordinances.py @@ -113,9 +113,6 @@ def main(): before = [row.copy() for row in result] standard = use_standard_types(result) validate_types(before, result) - shared = sum(Counter(row_key(row) for row in result).values()) - len( - {row_key(row) for row in result} - ) with output.open("w", encoding="utf-8-sig", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=columns) writer.writeheader() @@ -134,8 +131,7 @@ def main(): print(f"{filled} rows had an empty value; copied their summary into value.") print(f"{plain} rows had unit detail moved into their summary.") print( - f"{standard} rows had a local class name mapped to a standard type; " - f"{shared} rows now repeat another row's key." + f"{standard} rows had a local class name mapped to a standard type." ) print( f"Validated unique keys, unchanged singleton rows, integer years, and CSV round-trip: {output}" @@ -387,7 +383,6 @@ def merge_metadata(rows): STANDARD_TYPES = { "": "", "data processing center": "", - "data processing facility": "", "hyperscale data center": "Hyperscale Data Center", "large data center": "Large Data Center", "large-scale data center": "Large Data Center", @@ -401,6 +396,8 @@ def merge_metadata(rows): "associated data room or server room": "Accessory Data Center", "campus": "Data Center Campus", "large quantity water user": "Other", + # Albemarle defines this apart from data centers: staffed data entry + "data processing facility": "Other", } @@ -424,6 +421,8 @@ def use_standard_types(rows): def validate_types(before, after): """Only the type and summary changed, and every type is standard.""" allowed = set(STANDARD_TYPES.values()) + keys = Counter(row_key(row) for row in after) + assert max(keys.values()) == 1, "Two rows share a standard type and feature." for old, new in zip(before, after, strict=True): assert new["data_center_type"] in allowed if new == old: diff --git a/compass/extraction/data_centers/data_centers_schema.json b/compass/extraction/data_centers/data_centers_schema.json index 3a40a513f..3ca910aca 100644 --- a/compass/extraction/data_centers/data_centers_schema.json +++ b/compass/extraction/data_centers/data_centers_schema.json @@ -11,7 +11,7 @@ "properties": { "outputs": { "type": "array", - "description": "Sparse long-form extraction table. Emit a row only for a feature with an explicit, enacted requirement. Emit at most one row per feature and class of data center the ordinance names: where an ordinance sets a different standard for each class, emit one row per class rather than collapsing them.", + "description": "Sparse long-form extraction table. Emit a row only for a feature with an explicit, enacted requirement. Emit at most one row per (feature, data_center_type) pair; no two rows may share both. Where an ordinance sets a different standard for each class of data center, emit one row per data_center_type rather than collapsing them all.", "items": { "type": "object", "required": [ @@ -127,7 +127,7 @@ "string", "null" ], - "description": "A short summary of the requirement in your own words. Where data_center_type is not null, start with the jurisdiction's own name for the class, exactly as the ordinance writes it. Retain all specifics and conditions, such as an exception for a power outage, a standard that steps up next to a residential district, or a sunset date. Where the ordinance sets several values for this feature, state which one value holds and list every other value with the condition it applies under, such as the time period, receiving use, zoning district, street class, or yard. The verbatim excerpt belongs in ordinance_text. For qualitative features, state the requirement in full." + "description": "A short summary of the requirement in your own words. Where data_center_type is not null, start with the jurisdiction's own name for the class, exactly as the ordinance writes it, or with every such name where several local classes share the row. Retain all specifics and conditions, such as an exception for a power outage, a standard that steps up next to a residential district, or a sunset date. Where the ordinance sets several values for this feature, state which one value holds and list every other value with the condition it applies under, such as the time period, receiving use, zoning district, street class, or yard. The verbatim excerpt belongs in ordinance_text. For qualitative features, state the requirement in full." }, "ordinance_text": { "type": "string", @@ -264,7 +264,7 @@ }, "terms": { "definition": { - "description": "The jurisdiction's definition of a data center term, such as 'Data Center', 'Data Processing Center', or a size class like 'Hyperscale Data Center'. Emit one row per defined term, with data_center_type set to the standard class the term maps to, or null for the generic data center term. SUMMARY: start with the defined term as written. VALUE: the definition text, verbatim. UNITS: null. SECTION: where the term is defined. IGNORE: definitions of other uses that merely mention data processing or computing in passing, such as a business office that lists data processing among its activities." + "description": "The jurisdiction's definition of a data center term, such as 'Data Center', 'Data Processing Center', or a size class like 'Hyperscale Data Center'. Emit one row per data_center_type: the standard class the defined term maps to, or null for the generic data center term. Where several defined terms map to the same data_center_type, put them in one row, each definition preceded by its term. SUMMARY: start with the defined term or terms as written. VALUE: the definition text, verbatim. UNITS: null. SECTION: where the term is defined. IGNORE: definitions of other uses that merely mention data processing or computing in passing, such as a business office that lists data processing among its activities." } } }, @@ -277,12 +277,12 @@ "Do not infer a requirement from related context, and do not emit a row for a feature the source does not address." ], "size_tiering": [ - "Data center ordinances frequently set different standards for different classes of facility. When they do, emit one row per class the ordinance names, and do not merge tiered standards into a single row.", + "Data center ordinances frequently set different standards for different classes of facility. When they do, emit one row per data_center_type, and do not merge standards for different types into a single row.", "data_center_type holds a standard class, never the jurisdiction's wording. Map the local class by its name: 'Hyperscale' -> 'Hyperscale Data Center'; 'Large', 'Large-Scale', or 'Major' -> 'Large Data Center'; 'Medium' or 'Mid-Size' -> 'Medium Data Center'; 'Small' or 'Minor' -> 'Small Data Center'; 'Micro' or 'Edge' -> 'Micro Data Center'; accessory, ancillary, or incidental to another principal use -> 'Accessory Data Center'; 'Campus' or a multi-building data center development -> 'Data Center Campus'.", "Map by the name the ordinance gives the class, not by the size thresholds behind it: one jurisdiction's 'Large' may be smaller than another's 'Medium'.", "Use 'Other' for a specific class that fits none of the standard values, such as a class defined by whether the site is part of a unified development plan.", "Use null where one standard applies to data centers generally. Generic terms such as 'Data Center', 'DATA CENTER', 'Data Processing Center', or 'Data Processing Facility' are not a specific class.", - "Two local classes may share a standard value. Keep them as separate rows, and start each summary with the jurisdiction's own class name so the rows can be told apart." + "Each (feature, data_center_type) pair appears in at most one row. Where two local classes map to the same standard value, they share one row: for a numeric feature choose the most restrictive value across them by $instructions.numeric_prioritization, and in summary name each local class with the value and conditions that apply to it." ], "setbacks": [ "Setbacks are minimum separation distances. A setback row carries a single numeric distance and its unit.", @@ -290,10 +290,10 @@ "Where a setback applies to ancillary mechanical or electrical equipment under a different standard than the building, record it under 'setback for mechanical equipment'; where one distance covers both, keep it on the reference-object feature." ], "numeric_prioritization": [ - "A numeric feature gets one row per data center class, and its value is a single number: the most restrictive value the ordinance sets for that feature and class.", + "A numeric feature gets one row per data_center_type, and its value is a single number: the most restrictive value the ordinance sets for that feature and type.", "Restrictiveness rules: setbacks, minimum lot size, minimum vegetative cover, buffer width, and minimum reclaimed water share -> choose the largest value; noise, low-frequency noise, enclosure sound levels, vibration, illumination, thermal increase, maximum height, maximum building size, maximum lot coverage, floor-area share cap, and water consumption ceilings -> choose the smallest value.", "Values that differ by zoning district, time period, receiving use, street class, or yard compete for the same row. Do not emit a row per condition and do not combine the values into one string. List every value that was not selected in summary, with the condition it applies under.", - "Size classes are the exception: standards for different data center classes stay in separate rows, per size_tiering, and the most restrictive value is chosen within each class.", + "Data center types are the exception: standards for different data_center_type values stay in separate rows, per size_tiering, and the most restrictive value is chosen within each type.", "Compare values across units if needed to find the most restrictive one, but record the selected value in its own unit. Never convert the reported value." ], "units": [ @@ -314,7 +314,7 @@ "$qa_checklist": [ "All output rows use only the 35 allowed feature IDs.", "Every row includes a non-empty ordinance_text excerpt and an explanation linking that evidence to the feature and facility class.", - "Every data_center_type is one of the standard values or null. Generic requirements have null, tiered standards have one row per class, and each row with a non-null type names the jurisdiction's own class at the start of summary.", + "Every data_center_type is one of the standard values or null. No two rows share both feature and data_center_type. Generic requirements have null, tiered standards have one row per type, and each row with a non-null type names the jurisdiction's own class or classes at the start of summary.", "Numeric setbacks, lot standards, and building size rows carry numeric values and non-null units.", "Numeric rows carry a single number, the most restrictive value, with a plain unit, and summary lists the other values with their conditions.", "Units use only the canonical vocabulary in $instructions.units and carry no measurement detail.", diff --git a/compass/extraction/data_centers/plugin_config.yaml b/compass/extraction/data_centers/plugin_config.yaml index 9f0832992..08fec7b68 100644 --- a/compass/extraction/data_centers/plugin_config.yaml +++ b/compass/extraction/data_centers/plugin_config.yaml @@ -121,7 +121,9 @@ extraction_system_prompt: |- Data center ordinances frequently regulate by facility size class. Where a jurisdiction sets a different standard for each class, emit - one row per class rather than collapsing the tiers into one row. + one row per data_center_type rather than collapsing the tiers into one + row. No two rows may share both feature and data_center_type: local + classes that map to the same type share a row. For a numeric feature, value is a single number: the most restrictive value the ordinance sets for that feature and facility class, such as diff --git a/tests/python/unit/extraction/test_extraction_data_centers_schema.py b/tests/python/unit/extraction/test_extraction_data_centers_schema.py index fbb63d2a7..c8b42ff01 100644 --- a/tests/python/unit/extraction/test_extraction_data_centers_schema.py +++ b/tests/python/unit/extraction/test_extraction_data_centers_schema.py @@ -62,6 +62,8 @@ def test_data_center_type_is_a_fixed_list(): rules = " ".join(schema["$instructions"]["size_tiering"]) for name in spec["enum"][:-1]: assert f"'{name}'" in rules, name + # local classes sharing a type merge, so a row's key stays unique + assert "at most one row" in rules def test_numeric_features_ask_for_most_restrictive_value(): From ea2eec0d0245ab3d7ca8cbe43d81caaaf944a9cb Mon Sep 17 00:00:00 2001 From: Rajendra Adhikari Date: Mon, 9 Nov 2026 19:18:50 -0600 Subject: [PATCH 04/13] Make data center types size tiers plus accessory only --- collapse_human_dc_ordinances.py | 22 ++++++++++--- .../data_centers/data_centers_schema.json | 31 +++++++++---------- .../data_centers/plugin_config.yaml | 20 ++++++------ .../test_extraction_data_centers_schema.py | 12 +++---- 4 files changed, 49 insertions(+), 36 deletions(-) diff --git a/collapse_human_dc_ordinances.py b/collapse_human_dc_ordinances.py index d6f5b57b5..93fcbb6eb 100644 --- a/collapse_human_dc_ordinances.py +++ b/collapse_human_dc_ordinances.py @@ -110,6 +110,13 @@ def main(): before = [row.copy() for row in result] plain = use_plain_units(result) validate_units(before, result) + kept = [ + row + for row in result + if row["data_center_type"].strip().casefold() not in UNSUPPORTED_TYPES + ] + dropped = len(result) - len(kept) + result = kept before = [row.copy() for row in result] standard = use_standard_types(result) validate_types(before, result) @@ -123,7 +130,7 @@ def main(): assert list(reader) == result duplicates = sum(len(rows) > 1 for rows in groups.values()) print( - f"{len(original)} -> {len(result)} rows; {duplicates} duplicate groups resolved." + f"{len(original)} -> {len(result) + dropped} rows; {duplicates} duplicate groups resolved." ) print(f"{noise} noise rows reduced to their lowest listed limit.") print(f"{reduced} multi-value rows reduced to their most restrictive value.") @@ -133,6 +140,7 @@ def main(): print( f"{standard} rows had a local class name mapped to a standard type." ) + print(f"{dropped} rows left out: neither a size tier nor accessory.") print( f"Validated unique keys, unchanged singleton rows, integer years, and CSV round-trip: {output}" ) @@ -394,10 +402,16 @@ def merge_metadata(rows): "accessory data center": "Accessory Data Center", "ancillary": "Accessory Data Center", "associated data room or server room": "Accessory Data Center", - "campus": "Data Center Campus", - "large quantity water user": "Other", + # A water rule for data centers of any size, not a size tier + "large quantity water user": "", +} + +# Classes that are neither a size tier nor accessory: their rows are left +# out, as the extraction schema emits none for them. +UNSUPPORTED_TYPES = { + "campus", # Albemarle defines this apart from data centers: staffed data entry - "data processing facility": "Other", + "data processing facility", } diff --git a/compass/extraction/data_centers/data_centers_schema.json b/compass/extraction/data_centers/data_centers_schema.json index 3ca910aca..cbf5fa8f9 100644 --- a/compass/extraction/data_centers/data_centers_schema.json +++ b/compass/extraction/data_centers/data_centers_schema.json @@ -72,16 +72,14 @@ "string", "null" ], - "description": "The standard facility class the provision governs. Must be one of the enumerated values or null; do not write the jurisdiction's own class name here. Choose by the rules in $instructions.size_tiering. Use null when the requirement applies to data centers generally. The jurisdiction's own class name and any conditions belong in summary, and the complete source wording in ordinance_text.", + "description": "The size tier of data center the provision governs, or 'Accessory Data Center'. Must be one of the enumerated values or null; do not write the jurisdiction's own class name here. Choose by the rules in $instructions.size_tiering. Use null when the requirement applies to data centers of any size. Any other distinction the ordinance draws is a condition, not a type. The jurisdiction's own class name and any conditions belong in summary, and the complete source wording in ordinance_text.", "enum": [ - "Hyperscale Data Center", - "Large Data Center", - "Medium Data Center", - "Small Data Center", "Micro Data Center", + "Small Data Center", + "Medium Data Center", + "Large Data Center", + "Hyperscale Data Center", "Accessory Data Center", - "Data Center Campus", - "Other", null ] }, @@ -264,7 +262,7 @@ }, "terms": { "definition": { - "description": "The jurisdiction's definition of a data center term, such as 'Data Center', 'Data Processing Center', or a size class like 'Hyperscale Data Center'. Emit one row per data_center_type: the standard class the defined term maps to, or null for the generic data center term. Where several defined terms map to the same data_center_type, put them in one row, each definition preceded by its term. SUMMARY: start with the defined term or terms as written. VALUE: the definition text, verbatim. UNITS: null. SECTION: where the term is defined. IGNORE: definitions of other uses that merely mention data processing or computing in passing, such as a business office that lists data processing among its activities." + "description": "The jurisdiction's definition of a data center term, such as 'Data Center', 'Data Processing Center', or a size class like 'Hyperscale Data Center'. Emit one row per data_center_type: the size tier the defined term maps to, 'Accessory Data Center' for an accessory term, or null for the generic data center term. Where several defined terms map to the same data_center_type, put them in one row, each definition preceded by its term. Do not emit a row for a defined term that is neither a size tier, an accessory data center, nor the generic data center term, such as a data center campus. SUMMARY: start with the defined term or terms as written. VALUE: the definition text, verbatim. UNITS: null. SECTION: where the term is defined. IGNORE: definitions of other uses that merely mention data processing or computing in passing, such as a business office that lists data processing among its activities." } } }, @@ -277,12 +275,13 @@ "Do not infer a requirement from related context, and do not emit a row for a feature the source does not address." ], "size_tiering": [ - "Data center ordinances frequently set different standards for different classes of facility. When they do, emit one row per data_center_type, and do not merge standards for different types into a single row.", - "data_center_type holds a standard class, never the jurisdiction's wording. Map the local class by its name: 'Hyperscale' -> 'Hyperscale Data Center'; 'Large', 'Large-Scale', or 'Major' -> 'Large Data Center'; 'Medium' or 'Mid-Size' -> 'Medium Data Center'; 'Small' or 'Minor' -> 'Small Data Center'; 'Micro' or 'Edge' -> 'Micro Data Center'; accessory, ancillary, or incidental to another principal use -> 'Accessory Data Center'; 'Campus' or a multi-building data center development -> 'Data Center Campus'.", - "Map by the name the ordinance gives the class, not by the size thresholds behind it: one jurisdiction's 'Large' may be smaller than another's 'Medium'.", - "Use 'Other' for a specific class that fits none of the standard values, such as a class defined by whether the site is part of a unified development plan.", - "Use null where one standard applies to data centers generally. Generic terms such as 'Data Center', 'DATA CENTER', 'Data Processing Center', or 'Data Processing Facility' are not a specific class.", - "Each (feature, data_center_type) pair appears in at most one row. Where two local classes map to the same standard value, they share one row: for a numeric feature choose the most restrictive value across them by $instructions.numeric_prioritization, and in summary name each local class with the value and conditions that apply to it." + "Data center ordinances frequently set different standards for different size tiers of facility. When they do, emit one row per data_center_type, and do not merge standards for different types into a single row.", + "data_center_type is a size tier, never the jurisdiction's wording. The tiers run from smallest to largest: 'Micro Data Center', 'Small Data Center', 'Medium Data Center', 'Large Data Center', 'Hyperscale Data Center'. Map the local tier by its name: 'Micro' or 'Edge' -> 'Micro Data Center'; 'Small' or 'Minor' -> 'Small Data Center'; 'Medium' or 'Mid-Size' -> 'Medium Data Center'; 'Large', 'Large-Scale', or 'Major' -> 'Large Data Center'; 'Hyperscale' -> 'Hyperscale Data Center'.", + "Map by the name the ordinance gives the tier, not by the size thresholds behind it: one jurisdiction's 'Large' may be smaller than another's 'Medium'. The smallest tier has no lower bound and the largest no upper bound.", + "'Accessory Data Center' is the one type that is not a size: use it, whatever the size, for a data center that is accessory, ancillary, or incidental to another principal use.", + "Use null where a standard applies to data centers of any size. Generic terms such as 'Data Center', 'DATA CENTER', 'Data Processing Center', or 'Data Processing Facility' are not a size tier.", + "Any other distinction is a condition, not a type: whether the site is part of a unified development plan or a campus, its zoning district, its water or power use, and the like. Standards that differ by such a condition compete for the same row under $instructions.numeric_prioritization, with the other values and their conditions listed in summary.", + "Each (feature, data_center_type) pair appears in at most one row. Where two local tiers map to the same value, they share one row: for a numeric feature choose the most restrictive value across them, and in summary name each local tier with the value and conditions that apply to it." ], "setbacks": [ "Setbacks are minimum separation distances. A setback row carries a single numeric distance and its unit.", @@ -293,7 +292,7 @@ "A numeric feature gets one row per data_center_type, and its value is a single number: the most restrictive value the ordinance sets for that feature and type.", "Restrictiveness rules: setbacks, minimum lot size, minimum vegetative cover, buffer width, and minimum reclaimed water share -> choose the largest value; noise, low-frequency noise, enclosure sound levels, vibration, illumination, thermal increase, maximum height, maximum building size, maximum lot coverage, floor-area share cap, and water consumption ceilings -> choose the smallest value.", "Values that differ by zoning district, time period, receiving use, street class, or yard compete for the same row. Do not emit a row per condition and do not combine the values into one string. List every value that was not selected in summary, with the condition it applies under.", - "Data center types are the exception: standards for different data_center_type values stay in separate rows, per size_tiering, and the most restrictive value is chosen within each type.", + "Size tiers are the exception: standards for different data_center_type values stay in separate rows, per size_tiering, and the most restrictive value is chosen within each type.", "Compare values across units if needed to find the most restrictive one, but record the selected value in its own unit. Never convert the reported value." ], "units": [ @@ -314,7 +313,7 @@ "$qa_checklist": [ "All output rows use only the 35 allowed feature IDs.", "Every row includes a non-empty ordinance_text excerpt and an explanation linking that evidence to the feature and facility class.", - "Every data_center_type is one of the standard values or null. No two rows share both feature and data_center_type. Generic requirements have null, tiered standards have one row per type, and each row with a non-null type names the jurisdiction's own class or classes at the start of summary.", + "Every data_center_type is one of the standard values or null. No two rows share both feature and data_center_type. Generic requirements have null, tiered standards have one row per type, and each row with a non-null type names the jurisdiction's own class or classes at the start of summary. Distinctions other than size or accessory use appear only as conditions in summary.", "Numeric setbacks, lot standards, and building size rows carry numeric values and non-null units.", "Numeric rows carry a single number, the most restrictive value, with a plain unit, and summary lists the other values with their conditions.", "Units use only the canonical vocabulary in $instructions.units and carry no measurement detail.", diff --git a/compass/extraction/data_centers/plugin_config.yaml b/compass/extraction/data_centers/plugin_config.yaml index 08fec7b68..f5299fcf7 100644 --- a/compass/extraction/data_centers/plugin_config.yaml +++ b/compass/extraction/data_centers/plugin_config.yaml @@ -132,15 +132,17 @@ extraction_system_prompt: |- condition it applies under, such as the time period, receiving use, zoning district, street class, or yard. Keep units to the unit alone. - data_center_type takes only the schema's standard values. Map the - jurisdiction's class onto the closest one by its name, such as Large - Data Center for a Large-Scale or Major class, and use Other for a - specific class that fits none of them. Set it to null when the - requirement applies to data centers generally: generic labels such as - Data Center, DATA CENTER, and Data Processing Center are not specific - classes. Start summary with the jurisdiction's own class name whenever - data_center_type is not null, and retain the complete source wording - in ordinance_text. + data_center_type is a size tier from the schema's fixed list, or + Accessory Data Center. Map the jurisdiction's tier onto the list by + its name, such as Large Data Center for a Large-Scale or Major tier. + Set it to null when the requirement applies to data centers of any + size: generic labels such as Data Center, DATA CENTER, and Data + Processing Center are not size tiers. Any other distinction, such as + whether the site is part of a unified development plan, is a + condition rather than a type: take the most restrictive value and + list the others with their conditions in summary. Start summary with + the jurisdiction's own class name whenever data_center_type is not + null, and retain the complete source wording in ordinance_text. Do not extract text focused only on office, warehouse, distribution, telecommunications, or utility uses unless the same clause directly diff --git a/tests/python/unit/extraction/test_extraction_data_centers_schema.py b/tests/python/unit/extraction/test_extraction_data_centers_schema.py index c8b42ff01..66711b3b2 100644 --- a/tests/python/unit/extraction/test_extraction_data_centers_schema.py +++ b/tests/python/unit/extraction/test_extraction_data_centers_schema.py @@ -43,20 +43,18 @@ def test_schema_extracts_definitions_of_data_center_terms(): def test_data_center_type_is_a_fixed_list(): - """Types come from a closed list, so similar names cannot diverge""" + """Types are size tiers plus accessory, from a closed list""" schema = json.loads(SCHEMA_FP.read_text(encoding="utf-8")) spec = schema["properties"]["outputs"]["items"]["properties"][ "data_center_type" ] assert spec["enum"] == [ - "Hyperscale Data Center", - "Large Data Center", - "Medium Data Center", - "Small Data Center", "Micro Data Center", + "Small Data Center", + "Medium Data Center", + "Large Data Center", + "Hyperscale Data Center", "Accessory Data Center", - "Data Center Campus", - "Other", None, ] rules = " ".join(schema["$instructions"]["size_tiering"]) From bb09c45c7f6d283421bce08e9915e248d67648b3 Mon Sep 17 00:00:00 2001 From: Rajendra Adhikari Date: Mon, 9 Nov 2026 20:27:33 -0600 Subject: [PATCH 05/13] Map data center tiers by their own name, not their rank --- compass/extraction/data_centers/data_centers_schema.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compass/extraction/data_centers/data_centers_schema.json b/compass/extraction/data_centers/data_centers_schema.json index cbf5fa8f9..5c280f121 100644 --- a/compass/extraction/data_centers/data_centers_schema.json +++ b/compass/extraction/data_centers/data_centers_schema.json @@ -276,8 +276,8 @@ ], "size_tiering": [ "Data center ordinances frequently set different standards for different size tiers of facility. When they do, emit one row per data_center_type, and do not merge standards for different types into a single row.", - "data_center_type is a size tier, never the jurisdiction's wording. The tiers run from smallest to largest: 'Micro Data Center', 'Small Data Center', 'Medium Data Center', 'Large Data Center', 'Hyperscale Data Center'. Map the local tier by its name: 'Micro' or 'Edge' -> 'Micro Data Center'; 'Small' or 'Minor' -> 'Small Data Center'; 'Medium' or 'Mid-Size' -> 'Medium Data Center'; 'Large', 'Large-Scale', or 'Major' -> 'Large Data Center'; 'Hyperscale' -> 'Hyperscale Data Center'.", - "Map by the name the ordinance gives the tier, not by the size thresholds behind it: one jurisdiction's 'Large' may be smaller than another's 'Medium'. The smallest tier has no lower bound and the largest no upper bound.", + "data_center_type is a size tier, never the jurisdiction's wording. Where the ordinance's own tier name contains one of the words Micro, Small, Medium, Large, or Hyperscale, use the value with that same word: a 'Hyperscale Data Center' is always 'Hyperscale Data Center' and a 'Micro Data Center' is always 'Micro Data Center', however many tiers the ordinance has. Only a tier named with none of those words is mapped by meaning: 'Edge' -> 'Micro Data Center'; 'Minor' -> 'Small Data Center'; 'Mid-Size' -> 'Medium Data Center'; 'Large-Scale' or 'Major' -> 'Large Data Center'.", + "Map by the name the ordinance gives the tier, never by its size thresholds or its rank among the jurisdiction's tiers: one jurisdiction's 'Large' may be smaller than another's 'Medium', and a jurisdiction's largest tier is not 'Large Data Center' unless it is named so.", "'Accessory Data Center' is the one type that is not a size: use it, whatever the size, for a data center that is accessory, ancillary, or incidental to another principal use.", "Use null where a standard applies to data centers of any size. Generic terms such as 'Data Center', 'DATA CENTER', 'Data Processing Center', or 'Data Processing Facility' are not a size tier.", "Any other distinction is a condition, not a type: whether the site is part of a unified development plan or a campus, its zoning district, its water or power use, and the like. Standards that differ by such a condition compete for the same row under $instructions.numeric_prioritization, with the other values and their conditions listed in summary.", From bef2e83cadea559fb31bcb32428153e36b9c6cba Mon Sep 17 00:00:00 2001 From: Rajendra Adhikari Date: Mon, 9 Nov 2026 20:43:08 -0600 Subject: [PATCH 06/13] Name the jurisdiction and skip drafts and other jurisdictions' rules in extraction --- .../data_centers/data_centers_schema.json | 6 ++++-- .../data_centers/plugin_config.yaml | 19 ++++++++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/compass/extraction/data_centers/data_centers_schema.json b/compass/extraction/data_centers/data_centers_schema.json index 5c280f121..27bc666e3 100644 --- a/compass/extraction/data_centers/data_centers_schema.json +++ b/compass/extraction/data_centers/data_centers_schema.json @@ -272,6 +272,8 @@ "Also record each definition of a data center term as a 'definition' row, even though a definition is not itself a requirement.", "Write summary in your own words. Put verbatim source passages in ordinance_text and explain how they support the row in explanation.", "Never convert units. Preserve the jurisdiction's vocabulary except for the standard data_center_type values and the canonical unit names. Keep ordinance_text verbatim.", + "Extract only what the jurisdiction named in the system message has itself enacted. A meeting packet or staff report may reproduce other jurisdictions' regulations for comparison or attach text labelled as a draft or proposal: neither is a requirement of this jurisdiction, so emit no row from them.", + "Additions and strikethroughs alone do not make a text a draft. Unless it is labelled a draft or proposal, or says it was not adopted, extract an amending ordinance as amended: keep the added wording and leave out the struck wording.", "Do not infer a requirement from related context, and do not emit a row for a feature the source does not address." ], "size_tiering": [ @@ -281,7 +283,7 @@ "'Accessory Data Center' is the one type that is not a size: use it, whatever the size, for a data center that is accessory, ancillary, or incidental to another principal use.", "Use null where a standard applies to data centers of any size. Generic terms such as 'Data Center', 'DATA CENTER', 'Data Processing Center', or 'Data Processing Facility' are not a size tier.", "Any other distinction is a condition, not a type: whether the site is part of a unified development plan or a campus, its zoning district, its water or power use, and the like. Standards that differ by such a condition compete for the same row under $instructions.numeric_prioritization, with the other values and their conditions listed in summary.", - "Each (feature, data_center_type) pair appears in at most one row. Where two local tiers map to the same value, they share one row: for a numeric feature choose the most restrictive value across them, and in summary name each local tier with the value and conditions that apply to it." + "Each (feature, data_center_type) pair appears in at most one row. A rule stated in more than one section is one row whose section cites each of them. Where two local tiers map to the same value, they share one row: for a numeric feature choose the most restrictive value across them, and in summary name each local tier with the value and conditions that apply to it." ], "setbacks": [ "Setbacks are minimum separation distances. A setback row carries a single numeric distance and its unit.", @@ -321,7 +323,7 @@ "District rows are arrays of district names or codes, and jurisdiction-wide bans are recorded under 'prohibitions/moratorium' rather than 'prohibited use districts'.", "Screening, buffer, and fencing rows are assigned by what the provision governs: equipment, site perimeter planting, or perimeter fencing.", "Water rows are assigned by what the provision governs: technology, volume ceiling, or source substitution.", - "No row is emitted from a proposed, denied, or superseded provision." + "No row is emitted from a proposed, denied, or superseded provision, or from another jurisdiction's regulations reproduced in the text." ], "$qualitative_features": [ "parking", diff --git a/compass/extraction/data_centers/plugin_config.yaml b/compass/extraction/data_centers/plugin_config.yaml index f5299fcf7..4edea7630 100644 --- a/compass/extraction/data_centers/plugin_config.yaml +++ b/compass/extraction/data_centers/plugin_config.yaml @@ -112,6 +112,20 @@ extraction_system_prompt: |- - cooling technology, water consumption limits, and reclaimed water requirements + The jurisdiction is {jur_full_name}. Extract only requirements that + this jurisdiction has itself enacted. The text may be a meeting packet + or staff report rather than the code itself. Such a document often + attaches text labelled as a draft or proposal, or reproduces other + jurisdictions' regulations for comparison: emit no rows from either, + however detailed they are. If nothing else remains, return no rows + and say why in extraction_notes. + + Markup alone does not make a text a draft. Ordinances that amend a + code are commonly published showing additions and strikethroughs. + Unless the text is labelled a draft or proposal, or says it was not + adopted, extract it as amended: keep the added wording and leave out + the struck wording. + Extract only enacted local requirements that explicitly govern data centers, or that govern a use the ordinance defines as a data center, such as a data processing center or a computer server facility. The @@ -123,7 +137,10 @@ extraction_system_prompt: |- Where a jurisdiction sets a different standard for each class, emit one row per data_center_type rather than collapsing the tiers into one row. No two rows may share both feature and data_center_type: local - classes that map to the same type share a row. + classes that map to the same type share a row, and a rule the text + states in more than one section is one row that cites each section. + Before answering, check the rows for a repeated feature and + data_center_type pair and merge any you find. For a numeric feature, value is a single number: the most restrictive value the ordinance sets for that feature and facility class, such as From 44582b01b4d7b26663ca71886e897481e1cdda34 Mon Sep 17 00:00:00 2001 From: Rajendra Adhikari Date: Mon, 9 Nov 2026 20:43:20 -0600 Subject: [PATCH 07/13] Ask for the full definition in a definition row's summary --- compass/extraction/data_centers/data_centers_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compass/extraction/data_centers/data_centers_schema.json b/compass/extraction/data_centers/data_centers_schema.json index 27bc666e3..e63da624d 100644 --- a/compass/extraction/data_centers/data_centers_schema.json +++ b/compass/extraction/data_centers/data_centers_schema.json @@ -262,7 +262,7 @@ }, "terms": { "definition": { - "description": "The jurisdiction's definition of a data center term, such as 'Data Center', 'Data Processing Center', or a size class like 'Hyperscale Data Center'. Emit one row per data_center_type: the size tier the defined term maps to, 'Accessory Data Center' for an accessory term, or null for the generic data center term. Where several defined terms map to the same data_center_type, put them in one row, each definition preceded by its term. Do not emit a row for a defined term that is neither a size tier, an accessory data center, nor the generic data center term, such as a data center campus. SUMMARY: start with the defined term or terms as written. VALUE: the definition text, verbatim. UNITS: null. SECTION: where the term is defined. IGNORE: definitions of other uses that merely mention data processing or computing in passing, such as a business office that lists data processing among its activities." + "description": "The jurisdiction's definition of a data center term, such as 'Data Center', 'Data Processing Center', or a size class like 'Hyperscale Data Center'. Emit one row per data_center_type: the size tier the defined term maps to, 'Accessory Data Center' for an accessory term, or null for the generic data center term. Where several defined terms map to the same data_center_type, put them in one row, each definition preceded by its term. Do not emit a row for a defined term that is neither a size tier, an accessory data center, nor the generic data center term, such as a data center campus. SUMMARY: each defined term as written, followed by what it is defined to mean, in full, including any size or power thresholds. VALUE: the definition text, verbatim. UNITS: null. SECTION: where the term is defined. IGNORE: definitions of other uses that merely mention data processing or computing in passing, such as a business office that lists data processing among its activities." } } }, From 2c2135f8e14cc61986165a8d36156f07336959d5 Mon Sep 17 00:00:00 2001 From: Rajendra Adhikari Date: Tue, 10 Nov 2026 09:55:26 -0600 Subject: [PATCH 08/13] Type rows by how the ordinance states each requirement --- .../extraction/data_centers/data_centers_schema.json | 5 +++-- compass/extraction/data_centers/plugin_config.yaml | 12 +++++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/compass/extraction/data_centers/data_centers_schema.json b/compass/extraction/data_centers/data_centers_schema.json index e63da624d..4389769f3 100644 --- a/compass/extraction/data_centers/data_centers_schema.json +++ b/compass/extraction/data_centers/data_centers_schema.json @@ -277,11 +277,12 @@ "Do not infer a requirement from related context, and do not emit a row for a feature the source does not address." ], "size_tiering": [ - "Data center ordinances frequently set different standards for different size tiers of facility. When they do, emit one row per data_center_type, and do not merge standards for different types into a single row.", + "Follow the ordinance's own structure, feature by feature. Where it states a requirement separately for a size tier or for accessory data centers, emit a row with that data_center_type. Where it states a requirement for data centers generally, emit one row with null.", + "Do not split a generally stated requirement into a row per tier, and do not merge separately stated tier requirements into a null row, even when their wording is the same. One feature being stated per tier does not make every feature tiered: decide each feature from how the ordinance states that requirement.", "data_center_type is a size tier, never the jurisdiction's wording. Where the ordinance's own tier name contains one of the words Micro, Small, Medium, Large, or Hyperscale, use the value with that same word: a 'Hyperscale Data Center' is always 'Hyperscale Data Center' and a 'Micro Data Center' is always 'Micro Data Center', however many tiers the ordinance has. Only a tier named with none of those words is mapped by meaning: 'Edge' -> 'Micro Data Center'; 'Minor' -> 'Small Data Center'; 'Mid-Size' -> 'Medium Data Center'; 'Large-Scale' or 'Major' -> 'Large Data Center'.", "Map by the name the ordinance gives the tier, never by its size thresholds or its rank among the jurisdiction's tiers: one jurisdiction's 'Large' may be smaller than another's 'Medium', and a jurisdiction's largest tier is not 'Large Data Center' unless it is named so.", "'Accessory Data Center' is the one type that is not a size: use it, whatever the size, for a data center that is accessory, ancillary, or incidental to another principal use.", - "Use null where a standard applies to data centers of any size. Generic terms such as 'Data Center', 'DATA CENTER', 'Data Processing Center', or 'Data Processing Facility' are not a size tier.", + "Use null where the ordinance states a standard for data centers of any size. Generic terms such as 'Data Center', 'DATA CENTER', 'Data Processing Center', or 'Data Processing Facility' are not a size tier.", "Any other distinction is a condition, not a type: whether the site is part of a unified development plan or a campus, its zoning district, its water or power use, and the like. Standards that differ by such a condition compete for the same row under $instructions.numeric_prioritization, with the other values and their conditions listed in summary.", "Each (feature, data_center_type) pair appears in at most one row. A rule stated in more than one section is one row whose section cites each of them. Where two local tiers map to the same value, they share one row: for a numeric feature choose the most restrictive value across them, and in summary name each local tier with the value and conditions that apply to it." ], diff --git a/compass/extraction/data_centers/plugin_config.yaml b/compass/extraction/data_centers/plugin_config.yaml index 4edea7630..d9bd0525f 100644 --- a/compass/extraction/data_centers/plugin_config.yaml +++ b/compass/extraction/data_centers/plugin_config.yaml @@ -134,9 +134,15 @@ extraction_system_prompt: |- technology-specific title. Data center ordinances frequently regulate by facility size class. - Where a jurisdiction sets a different standard for each class, emit - one row per data_center_type rather than collapsing the tiers into one - row. No two rows may share both feature and data_center_type: local + Follow the ordinance's own structure, feature by feature. Where it + states a requirement separately for a size tier, emit a row for that + data_center_type. Where it states a requirement for data centers + generally, emit one row with a null data_center_type. Do not split a + general requirement into a row per tier, and do not merge separately + stated tier requirements into one row. One feature being stated per + tier does not make every feature tiered. + + No two rows may share both feature and data_center_type: local classes that map to the same type share a row, and a rule the text states in more than one section is one row that cites each section. Before answering, check the rows for a repeated feature and From 64be1c2f23afc2f6178841af459da288af754642 Mon Sep 17 00:00:00 2001 From: Rajendra Adhikari Date: Tue, 10 Nov 2026 11:14:09 -0600 Subject: [PATCH 09/13] Fix CI: integration test expects zero-padded jurisdiction codes --- .../test_integrated_pipeline_orchestration.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/python/integration/test_integrated_pipeline_orchestration.py b/tests/python/integration/test_integrated_pipeline_orchestration.py index cbd447f66..d52665841 100644 --- a/tests/python/integration/test_integrated_pipeline_orchestration.py +++ b/tests/python/integration/test_integrated_pipeline_orchestration.py @@ -733,6 +733,7 @@ def test_collection_cli_selectively_resets_and_collects_missing_shards( "County,State\nWhatcom,Washington\nAdams,Colorado\nBoulder,Colorado\n", encoding="utf-8", ) + # keys without the catalog's leading zero still find their documents for code in ["8001", "8013"]: config["known_local_docs"][code] = config["known_local_docs"]["53073"] collection_cli_config.write_text(json.dumps(config), encoding="utf-8") @@ -745,14 +746,14 @@ def test_collection_cli_selectively_resets_and_collects_missing_shards( for path in output.rglob("*_collection_manifest.json") } passing_bytes = shards["53073"].read_bytes() - failed = json.loads(shards["8001"].read_text(encoding="utf-8")) + failed = json.loads(shards["08001"].read_text(encoding="utf-8")) failed.update( num_docs=0, results=[], completed_step_document_counts={"known_local_docs": 0}, ) - shards["8001"].write_text(json.dumps(failed), encoding="utf-8") - shards["8013"].unlink() + shards["08001"].write_text(json.dumps(failed), encoding="utf-8") + shards["08013"].unlink() seed_fp = output / "search_input.json" seed_fp.write_text('{"preserved": true}', encoding="utf-8") COMPASS_PB.reset() @@ -764,7 +765,7 @@ def test_collection_cli_selectively_resets_and_collects_missing_shards( assert result.exit_code == 0, f"{result.output}\n{result.exception}" assert shards["53073"].read_bytes() == passing_bytes - for code in ["8001", "8013"]: + for code in ["08001", "08013"]: assert ( json.loads(shards[code].read_text(encoding="utf-8"))["num_docs"] == 1 From c42fc750459c14f59d7ae8b9304aa26f7cefa69d Mon Sep 17 00:00:00 2001 From: Rajendra Adhikari Date: Tue, 10 Nov 2026 17:25:36 -0600 Subject: [PATCH 10/13] Keep the human-reference scripts out of the package --- collapse_human_dc_ordinances.py | 484 -------------------------------- export_human_dc_ordinances.py | 212 -------------- 2 files changed, 696 deletions(-) delete mode 100644 collapse_human_dc_ordinances.py delete mode 100644 export_human_dc_ordinances.py diff --git a/collapse_human_dc_ordinances.py b/collapse_human_dc_ordinances.py deleted file mode 100644 index 93fcbb6eb..000000000 --- a/collapse_human_dc_ordinances.py +++ /dev/null @@ -1,484 +0,0 @@ -"""Collapse duplicate ordinance keys using the agreed feature-specific rules.""" - -import argparse -import csv -import re -from collections import Counter, defaultdict -from decimal import Decimal -from pathlib import Path - -# Features the extraction schema records as text rather than a number -QUALITATIVE = { - "parking", - "equipment screening/enclosure", - "fencing/security screening", - "backup generator operating restrictions", - "cooling technology", -} - -# Most restrictive value picked by hand from each multi-value row, keyed by -# (jurisdiction, feature). None marks a row with no single value to pick, -# such as a berm-only buffer or a setback that only scales with height. -MOST_RESTRICTIVE = { - ("Phoenix", "buffer/landscaping"): ("30", "feet"), - ("Elk Grove Village", "Minimum Lot Size"): ("4", "acres"), - ("Elk Grove Village", "Maximum Height"): ("100", "feet"), - ("Elk Grove Village", "Setback - Property Line"): ("50", "feet"), - ("Elk Grove Village", "Light Illumination Limits"): ("0.5", "footcandles"), - ("Elk Grove Village", "Maximum Lot Coverage"): ("50", "percent"), - ("Tippecanoe", "Vibration"): ("0.002", "g peak"), - ("Tippecanoe", "Setback - Property Line"): ("40", "feet"), - ("Tippecanoe", "Setback - Roads"): ("60", "feet"), - ("Tippecanoe", "Minimum Lot Size"): ("10000", "sq ft"), - ("Tippecanoe", "buffer/landscaping"): None, - ("Lancaster City", "buffer/landscaping"): ("100", "feet"), - ("West Pennsboro Township", "Minimum Lot Size"): ("25", "acres"), - ("West Pennsboro Township", "Setback - Property Line"): ("150", "feet"), - ("West Pennsboro Township", "Maximum Lot Coverage"): ("70", "percent"), - ("Loudoun", "Setback - Residential"): ("200", "feet"), - ("Loudoun", "buffer/landscaping"): None, - ("Prince William", "noise (low-frequency/tonal)"): ("68", "dB(C)"), - ("Prince William", "buffer/landscaping"): None, - ("Manassas City", "Setback - Property Line"): ("25", "feet"), - ("Manassas City", "Setback - Residential"): None, - ("Manassas City", "buffer/landscaping"): ("25", "feet"), - ("Albemarle", "noise mitigation/acoustic barriers"): ("70", "dBA"), -} - -# Units recorded with measurement detail, mapped to the unit alone. The -# detail moves into the summary, matching the extraction schema. -PLAIN_UNITS = { - "% max building coverage; no FAR standard": "percent", - "% minimum vegetative cover of lot": "percent", - "FAR (max)": "FAR", - "feet (max)": "feet", - "feet (up to)": "feet", - "feet from all lot lines": "feet", - "feet from listed receptors; measured property line to property line": "feet", - "feet from major/minor arterial; at least one arterial access required": "feet", - "feet from residential/community lot lines and residentially zoned or PO districts": "feet", - "feet from the Rural Areas zoning district": "feet", - "feet max building height": "feet", - "footcandles in residential or RE zone": "footcandles", - "percent annual cap": "percent", - "percent lot coverage": "percent", - "percent minimum reclaimed-water offset of total water use (director may set another percentage based on proximity to the reclaimed system)": "percent", - "sq ft GFA": "sq ft", - "sq ft/floor": "sq ft", - "°F maximum increase at zone boundary line": "°F", -} - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "input", - nargs="?", - type=Path, - default=Path(__file__).with_name("human_extracted_dc_ordinances.csv"), - ) - parser.add_argument("--output", type=Path) - args = parser.parse_args() - output = args.output or args.input.with_name( - "human_extracted_dc_ordinances_collapsed.csv" - ) - if output.resolve() == args.input.resolve(): - raise ValueError("Use a separate output file to preserve the original CSV.") - with args.input.open(encoding="utf-8-sig", newline="") as handle: - reader = csv.DictReader(handle) - columns = reader.fieldnames - original = list(reader) - groups = defaultdict(list) - for row in original: - groups[row_key(row)].append(row) - result = [] - for rows in groups.values(): - result.extend(collapse(rows) if len(rows) > 1 else rows) - validate(result, groups) - before = [row.copy() for row in result] - noise = select_lowest_noise_limits(result) - validate_noise(before, result) - before = [row.copy() for row in result] - reduced = select_most_restrictive_values(result) - validate_most_restrictive(before, result) - before = [row.copy() for row in result] - qualitative = use_summary_for_qualitative(result) - validate_qualitative(before, result) - before = [row.copy() for row in result] - filled = fill_value_from_summary(result) - validate_fill(before, result) - before = [row.copy() for row in result] - plain = use_plain_units(result) - validate_units(before, result) - kept = [ - row - for row in result - if row["data_center_type"].strip().casefold() not in UNSUPPORTED_TYPES - ] - dropped = len(result) - len(kept) - result = kept - before = [row.copy() for row in result] - standard = use_standard_types(result) - validate_types(before, result) - with output.open("w", encoding="utf-8-sig", newline="") as handle: - writer = csv.DictWriter(handle, fieldnames=columns) - writer.writeheader() - writer.writerows(result) - with output.open(encoding="utf-8-sig", newline="") as handle: - reader = csv.DictReader(handle) - assert reader.fieldnames == columns - assert list(reader) == result - duplicates = sum(len(rows) > 1 for rows in groups.values()) - print( - f"{len(original)} -> {len(result) + dropped} rows; {duplicates} duplicate groups resolved." - ) - print(f"{noise} noise rows reduced to their lowest listed limit.") - print(f"{reduced} multi-value rows reduced to their most restrictive value.") - print(f"{qualitative} qualitative rows now carry their summary as value.") - print(f"{filled} rows had an empty value; copied their summary into value.") - print(f"{plain} rows had unit detail moved into their summary.") - print( - f"{standard} rows had a local class name mapped to a standard type." - ) - print(f"{dropped} rows left out: neither a size tier nor accessory.") - print( - f"Validated unique keys, unchanged singleton rows, integer years, and CSV round-trip: {output}" - ) - - -def collapse(rows): - feature = rows[0]["feature"] - if feature in {"equipment screening/enclosure", "Cooling Technology"}: - row = merge_metadata(rows) - row.update(value=row["summary"], units="str") - return [row] - if feature == "Maximum Building Size": - assert len({r["units"] for r in rows}) == 1, ( - "Size limits must use comparable units." - ) - row = merge_metadata(rows) - row["value"] = str(min(Decimal(r["value"]) for r in rows)) - row["summary"] = ( - f"Selected most restrictive listed maximum: {row['value']} {row['units']}. " - + row["summary"] - ) - return [row] - if feature == "Noise": - # keep every listed limit; select_lowest_noise_limits picks one later - row = merge_metadata(rows) - row["value"] = "; ".join(r["value"].strip() for r in rows) - row["units"] = "; ".join(r["units"].strip() for r in rows) - return [row] - if feature == "definition": - return [collapse_definition(rows)] - raise ValueError(f"No collapse rule for duplicate feature: {feature}") - - -def select_lowest_noise_limits(rows): - """Keep the lowest listed limit as each Noise row's value. - - This matches the extraction schema, which records the single most - restrictive noise limit and leaves the other limits to the summary. - """ - selected = 0 - for row in rows: - if row["feature"] != "Noise": - continue - limit = min(listed_noise_limits(row["value"])) - row["summary"] = ( - f"Selected most restrictive listed limit: {limit} dBA " - f"(listed: {row['value']} {row['units']}). " + row["summary"] - ) - row.update(value=str(limit), units="dBA") - selected += 1 - return selected - - -def listed_noise_limits(value): - """Numbers in a noise value, skipping '+N' exception allowances. - - A segment such as '+10 for 15 min' raises the limit for a short - duration rather than setting one, so none of its numbers are limits. - """ - return [ - Decimal(number) - for segment in value.split(";") - if not segment.strip().startswith("+") - for number in re.findall(r"\d+(?:\.\d+)?", segment) - ] - - -def validate_noise(before, after): - """Only Noise rows changed, each to the lowest of its listed limits.""" - assert len(before) == len(after) - for old, new in zip(before, after, strict=True): - if old["feature"] != "Noise": - assert new == old, "A non-noise row was changed." - continue - assert Decimal(new["value"]) == min(listed_noise_limits(old["value"])) - assert new["units"] == "dBA" - assert new["summary"].endswith(old["summary"]) - rewritten = {"value", "units", "summary"} - assert all(new[k] == v for k, v in old.items() if k not in rewritten) - - -def collapse_definition(rows): - """Keep the adopted definition primary and label denied text as historical.""" - adopted = [row for row in rows if "ADOPTED/CURRENT." in row["explanation"]] - assert len(adopted) == 1, "A single adopted definition must be identified." - row = adopted[0].copy() - historical = [r for r in rows if r is not adopted[0]] - assert all("PROPOSED/DENIED." in r["explanation"] for r in historical) - row["summary"] = ( - "Uses the adopted July 7, 2026 moratorium Resolution definition, which draws no " - "size distinctions and applies for the moratorium period through March 30, 2027, " - "as recorded in the workbook. The denied 2026 draft definition is retained in " - "explanation as historical text and is not part of the selected definition." - ) - notes = ["Original adopted-definition notes:\n" + row["explanation"].strip()] - for old in historical: - notes.append( - "Historical proposed/denied definition (not operative):\n" - + old["value"] - + f"\nSection: {old['section']}\nYear: {old['year']}\nSource: {old['source']}" - + "\nOriginal denied-draft notes:\n" - + old["explanation"].strip() - ) - row.update(explanation="\n\n".join(notes), units="str") - return row - - -def select_most_restrictive_values(rows): - """Reduce each multi-value row to the value picked in MOST_RESTRICTIVE. - - The listed values are kept at the front of the summary, matching the - extraction schema, which records one value and leaves the rest to the - summary. - """ - matched = set() - for row in rows: - key = (row["subdivision"].strip() or row["county"].strip(), row["feature"]) - if key not in MOST_RESTRICTIVE: - continue - assert key not in matched, f"More than one row for {key}" - matched.add(key) - listed = f"(listed: {row['value']} {row['units']})" - picked = MOST_RESTRICTIVE[key] - if picked is None: - row["summary"] = f"No single most restrictive value {listed}. " + row["summary"] - row.update(value=row["summary"], units="str") - else: - value, units = picked - row["summary"] = ( - f"Selected most restrictive listed value: {value} {units} {listed}. " - + row["summary"] - ) - row.update(value=value, units=units) - assert matched == set(MOST_RESTRICTIVE), set(MOST_RESTRICTIVE) - matched - return len(matched) - - -def validate_most_restrictive(before, after): - """Only picked rows changed, each to a number its old value listed.""" - assert len(before) == len(after) - for old, new in zip(before, after, strict=True): - if new == old: - continue - key = (old["subdivision"].strip() or old["county"].strip(), old["feature"]) - assert key in MOST_RESTRICTIVE, f"An unpicked row was changed: {key}" - assert new["summary"].endswith(old["summary"]) - if MOST_RESTRICTIVE[key] is not None: - assert new["value"] in old["value"].replace(",", "") - rewritten = {"value", "units", "summary"} - assert all(new[k] == v for k, v in old.items() if k not in rewritten) - - -def use_plain_units(rows): - """Keep only the unit in units and move its detail into the summary.""" - changed = 0 - for row in rows: - units = row["units"].strip() - if units not in PLAIN_UNITS: - continue - row["summary"] = f"Units detail: {units}. " + row["summary"] - row["units"] = PLAIN_UNITS[units] - changed += 1 - return changed - - -def validate_units(before, after): - """Only detailed units changed, and every numeric row has a plain unit.""" - assert len(before) == len(after) - for old, new in zip(before, after, strict=True): - if new == old: - continue - units = old["units"].strip() - assert new == { - **old, - "units": PLAIN_UNITS[units], - "summary": f"Units detail: {units}. " + old["summary"], - } - allowed = set(PLAIN_UNITS.values()) | {"str", "acres", "dBA", "dB(C)", "g peak"} - assert all(row["units"] in allowed for row in after), { - row["units"] for row in after - } - allowed - - -def use_summary_for_qualitative(rows): - """Carry the summary as value for features the schema extracts as text. - - This follows the convention finalize applies to qualitative features. - """ - changed = 0 - for row in rows: - if row["feature"].casefold() not in QUALITATIVE: - continue - summary = row["summary"].strip() - if summary and (row["value"], row["units"]) != (summary, "str"): - row.update(value=summary, units="str") - changed += 1 - return changed - - -def validate_qualitative(before, after): - """Only qualitative rows changed, and only their value and units.""" - assert len(before) == len(after) - for old, new in zip(before, after, strict=True): - if new == old: - continue - assert old["feature"].casefold() in QUALITATIVE - assert new == {**old, "value": old["summary"].strip(), "units": "str"} - - -def fill_value_from_summary(rows): - """Use the summary as the value when the value is empty but a summary exists. - - Free-text answers are sometimes recorded only in ``summary``; an empty - ``value`` would otherwise read as "nothing found" downstream. - """ - filled = 0 - for row in rows: - if not row["value"].strip() and row["summary"].strip(): - row["value"] = row["summary"].strip() - filled += 1 - return filled - - -def validate_fill(before, after): - """Only empty values with a summary changed, and only their value.""" - assert len(before) == len(after) - for old, new in zip(before, after, strict=True): - if old["value"].strip() or not old["summary"].strip(): - assert new == old, "A row with a value (or no summary) was changed." - else: - assert new == {**old, "value": old["summary"].strip()} - assert all(row["value"].strip() for row in after if row["summary"].strip()) - - -def merge_metadata(rows): - row = rows[0].copy() - assert len({r["year"] for r in rows}) == 1 - for field in ("section", "summary", "explanation", "source"): - separator = " " if field == "summary" else "\n\n" - row[field] = separator.join( - dict.fromkeys(r[field].strip() for r in rows if r[field].strip()) - ) - return row - - -# The fixed list the extraction schema allows for data_center_type, keyed -# by the class names the workbook uses. An empty type is a generic one. -STANDARD_TYPES = { - "": "", - "data processing center": "", - "hyperscale data center": "Hyperscale Data Center", - "large data center": "Large Data Center", - "large-scale data center": "Large Data Center", - "major": "Large Data Center", - "medium data center": "Medium Data Center", - "small urban data center": "Small Data Center", - "minor": "Small Data Center", - "micro data center": "Micro Data Center", - "accessory data center": "Accessory Data Center", - "ancillary": "Accessory Data Center", - "associated data room or server room": "Accessory Data Center", - # A water rule for data centers of any size, not a size tier - "large quantity water user": "", -} - -# Classes that are neither a size tier nor accessory: their rows are left -# out, as the extraction schema emits none for them. -UNSUPPORTED_TYPES = { - "campus", - # Albemarle defines this apart from data centers: staffed data entry - "data processing facility", -} - - -def use_standard_types(rows): - """Map each local class name to a standard type, keeping it in summary. - - Runs last: the steps above tell rows apart by their local class name. - """ - changed = 0 - for row in rows: - local = row["data_center_type"].strip() - standard = STANDARD_TYPES[local.casefold()] - if standard == local: - continue - row["data_center_type"] = standard - row["summary"] = f"{local}: {row['summary']}" - changed += 1 - return changed - - -def validate_types(before, after): - """Only the type and summary changed, and every type is standard.""" - allowed = set(STANDARD_TYPES.values()) - keys = Counter(row_key(row) for row in after) - assert max(keys.values()) == 1, "Two rows share a standard type and feature." - for old, new in zip(before, after, strict=True): - assert new["data_center_type"] in allowed - if new == old: - continue - local = old["data_center_type"].strip() - assert new["summary"] == f"{local}: {old['summary']}" - rewritten = {"data_center_type", "summary"} - assert all(new[k] == v for k, v in old.items() if k not in rewritten) - - -def validate(result, groups): - counts = Counter(row_key(row) for row in result) - assert all(count == 1 for count in counts.values()) - assert all(not row["year"] or row["year"].isdigit() for row in result) - assert all( - row["data_center_type"].lower() not in {"all", "data center"} for row in result - ) - by_key = {row_key(row): row for row in result} - for key, rows in groups.items(): - if len(rows) == 1: - assert by_key[key] == rows[0], "A singleton row was changed." - elif rows[0]["feature"] in { - "equipment screening/enclosure", - "Cooling Technology", - }: - merged = by_key[key] - assert merged["units"] == "str" and merged["value"] - assert all(row["summary"].strip() in merged["value"] for row in rows) - assert len(result) == len(groups) - - -def row_key(row): - fields = ( - "state", - "county", - "subdivision", - "jurisdiction_type", - "data_center_type", - "feature", - ) - return tuple(row[field].strip().casefold() for field in fields) - - -if __name__ == "__main__": - main() diff --git a/export_human_dc_ordinances.py b/export_human_dc_ordinances.py deleted file mode 100644 index 8c97db3ad..000000000 --- a/export_human_dc_ordinances.py +++ /dev/null @@ -1,212 +0,0 @@ -import argparse -import csv -import re -from collections import Counter, defaultdict -from pathlib import Path -from urllib.parse import urlsplit - -import openpyxl - - -def main(): - parser = argparse.ArgumentParser() - base = Path(__file__).resolve().parent - parser.add_argument("--workbook", type=Path, default=base / "COMPASS DC Ordinance Research_2026.08.16.xlsx") - parser.add_argument("--reference", type=Path, default=base / "examples/one_shot_schema_extraction_dc/outputs/ordinances.csv") - parser.add_argument("--output", type=Path, default=base / "human_extracted_dc_ordinances.csv") - args = parser.parse_args() - workbook = openpyxl.load_workbook(args.workbook, data_only=True) - ordinances = sheet_rows(workbook["DC Ordinances"]) - definitions = sheet_rows(workbook["Definitions"]) - records = [ordinance_record(row) for row in ordinances] - records.extend(definition_record(row) for row in definitions) - assign_jurisdiction_metadata(records, base / "compass/data/conus_jurisdictions.csv") - with args.reference.open(encoding="utf-8-sig", newline="") as stream: - fields = [ - field for field in next(csv.reader(stream)) - if field in records[0] and field != "FIPS" - ] - fields.append("FIPS") - with args.output.open("w", encoding="utf-8-sig", newline="") as stream: - writer = csv.DictWriter(stream, fieldnames=fields) - writer.writeheader() - writer.writerows(records) - verify_export(args.output, fields, records, ordinances, definitions) - print(f"Wrote {len(records)} rows: {len(ordinances)} ordinance features and {len(definitions)} definitions.") - print("Data center types:", dict(Counter(record["data_center_type"] for record in records))) - for sheet in workbook["DC Ordinances"], workbook["Definitions"]: - report_link_mismatches(sheet) - - -def assign_jurisdiction_metadata(records, catalog_path): - """Resolve catalog types and FIPS using names and explicit name suffixes.""" - locations = defaultdict(list) - with catalog_path.open(encoding="utf-8-sig", newline="") as stream: - for row in csv.DictReader(stream): - locations[(row["State"].casefold(), row["Subdivision"].casefold())].append(row) - - for record in records: - state = record["state"].strip().casefold() - subdivision = (record["subdivision"] or "").strip().casefold() - county = (record["county"] or "").strip().casefold() - candidates = locations[(state, subdivision)] - if not candidates: - suffix = re.fullmatch(r"(.+) (city|town|township|village|borough)", subdivision) - if suffix: - candidates = [ - row for row in locations[(state, suffix[1])] - if row["Jurisdiction Type"].casefold() == suffix[2] - ] - if not subdivision: - candidates = [row for row in candidates if row["County"].casefold() == county] - elif county: - # Some catalog municipalities have no county recorded. - candidates = [row for row in candidates if row["County"].casefold() in {"", county}] - if len(candidates) != 1: - raise ValueError( - f"Expected one catalog match for {(state, county, subdivision)}, " - f"found {len(candidates)}" - ) - record["jurisdiction_type"] = candidates[0]["Jurisdiction Type"].title() - record["FIPS"] = candidates[0]["FIPS"] - - -def ordinance_record(row): - record = common_record(row) - record.update( - feature=normalize_feature(row["Feature"].value), - data_center_type=normalize_type(row["Data Center Type"].value), - value=row["Value"].value, - units="str" if row["Feature Data Type"].value == "Qualitative" else row["Units"].value, - summary=row["Summary"].value, - ) - return record - - -def definition_record(row): - record = common_record(row) - term = row["Defined Term"].value - record.update( - feature="definition", - data_center_type=normalize_type(term), - value=row["Definition (verbatim)"].value, - units="str", - summary=None, - explanation=f"Defined term: {term}.\n{record['explanation']}".rstrip(), - ) - return record - - -def common_record(row): - mapping = { - "county": "County", - "state": "State", - "subdivision": "City / Subdivision", - "jurisdiction_type": "Jurisdiction Type", - "section": "Section (Pin Cite)", - } - record = {key: row[column].value for key, column in mapping.items()} - raw_year = row["Ordinance Year"].value - year = int(re.match(r"^\s*(\d{4})\b", str(raw_year)).group(1)) - notes = row["Notes"].value or "" - # Keep adoption, repeal, and proposal annotations outside the numeric year. - if str(raw_year).strip() != str(year): - notes = f"{notes}\nOrdinance Year in workbook: {raw_year}".strip() - pdf = row.get("Direct PDF URL") - selected = pdf if pdf is not None and (pdf.value or pdf.hyperlink) else row["Source URL"] - record.update(year=year, explanation=notes, source=cell_url(selected)) - return record - - -def normalize_feature(value): - aliases = { - 'buffer / landscaping': 'buffer/landscaping', - 'equipment screening / enclosure': 'equipment screening/enclosure', - 'fencing / security screening': 'fencing/security screening', - 'heat / thermal impact': 'heat/thermal impact', - 'noise (low-frequency / tonal)': 'noise (low-frequency/tonal)', - 'noise mitigation / acoustic barriers': 'noise mitigation/acoustic barriers', - 'prohibitions / moratorium': 'prohibitions/moratorium', - 'setback - parks / natural areas': 'setback - parks/natural areas', - 'water reuse / reclaimed': 'water reuse/reclaimed', - } - return aliases.get(value.strip().casefold(), value) - - -def normalize_type(value): - if value is None: - return "" - term = value.strip() - # Parenthetical annotations describe the instrument, not a facility category. - if re.fullmatch(r"data centers?(?:\s*\(.*\))?", term, re.IGNORECASE): - return "" - if term.casefold() in {"all", "all types", "all data center types", "hyperscale / medium / micro"}: - return "" - aliases = { - "hyperscale": "Hyperscale Data Center", - "medium": "Medium Data Center", - "micro": "Micro Data Center", - "accessory": "Accessory Data Center", - "data center, hyperscale": "Hyperscale Data Center", - "data center, medium": "Medium Data Center", - "data center, micro": "Micro Data Center", - "data center, accessory": "Accessory Data Center", - "accessory use": "Accessory Data Center", - "data center as an accessory use": "Accessory Data Center", - "data center, accessory use": "Accessory Data Center", - "data center, ancillary": "Ancillary", - "data center, campus": "Campus", - "data center, major": "Major", - "data center, minor": "Minor", - } - return aliases.get(term.casefold(), term) - - -def verify_export(path, fields, records, ordinances, definitions): - with path.open(encoding="utf-8-sig", newline="") as stream: - reader = csv.DictReader(stream) - saved = list(reader) - assert reader.fieldnames == fields - assert len(saved) == len(ordinances) + len(definitions) - for actual, expected in zip(saved, records): - assert all(actual[key] == ("" if expected[key] is None else str(expected[key])) for key in fields) - assert type(expected["year"]) is int - assert re.fullmatch(r"\d{4}", actual["year"]) - assert actual["source"].startswith(("http://", "https://")) - assert not re.fullmatch(r"data center(?:\s*\(.*\))?", actual["data_center_type"], re.IGNORECASE) - for actual, row in zip(saved, ordinances): - if row["Feature Data Type"].value == "Qualitative": - assert actual["units"] == "str" - else: - units = row["Units"].value - assert actual["units"] == ("" if units is None else str(units)) - for actual, row in zip(saved[len(ordinances):], definitions): - assert actual["feature"] == "definition" and actual["units"] == "str" - assert actual["value"] == row["Definition (verbatim)"].value - print("Verified column order, all exported values, numeric years, qualitative units, and definition text.") - - -def sheet_rows(sheet): - headers = [cell.value for cell in sheet[1]] - return [ - dict(zip(headers, row)) - for row in sheet.iter_rows(min_row=2) - if any(cell.value is not None for cell in row) - ] - - -def cell_url(cell): - return cell.hyperlink.target if cell.hyperlink and cell.hyperlink.target else cell.value - - -def report_link_mismatches(sheet): - for row in sheet_rows(sheet): - cell = row["Source URL"] - visible = str(cell.value or "") - target = cell_url(cell) - if cell.hyperlink and visible.startswith("http") and urlsplit(visible).netloc != urlsplit(target).netloc: - print(f"Hyperlink host mismatch at {sheet.title}!{cell.coordinate}: {visible!r} -> {target!r}") - - -if __name__ == "__main__": - main() From 34d713dce99012b4bfa72b0d2a08a99cb60179c2 Mon Sep 17 00:00:00 2001 From: Rajendra Adhikari Date: Thu, 12 Nov 2026 13:33:32 -0600 Subject: [PATCH 11/13] Address review: match saved manifests by unpadded code, keep supplied qualitative values --- compass/pipeline/coordinator.py | 35 +++++++++------ compass/utilities/finalize.py | 17 +++++--- .../pipeline/test_pipeline_orchestration.py | 43 +++++++++++++++++++ .../unit/utilities/test_utilities_finalize.py | 30 +++++++++++++ 4 files changed, 107 insertions(+), 18 deletions(-) diff --git a/compass/pipeline/coordinator.py b/compass/pipeline/coordinator.py index fad516e56..5cf8cd907 100644 --- a/compass/pipeline/coordinator.py +++ b/compass/pipeline/coordinator.py @@ -492,7 +492,9 @@ async def run(self, jurisdictions_df): tasks = [] start_date = datetime.now(UTC) for jurisdiction in jurisdictions_from_df(jurisdictions_df): - collection_info = collection_infos_by_fips.get(jurisdiction.code) + collection_info = _known_source_for_code( + collection_infos_by_fips, fips_to_str(jurisdiction.code) + ) if collection_info is None: logger.warning( "No collection info found for %s; skipping extraction", @@ -622,18 +624,20 @@ async def _jurisdictions_with_search_input(runtime, jurisdictions_df): known_search_results = await _load_known_se_results(runtime) for jurisdiction in jurisdictions_from_df(jurisdictions_df): - if known_search_results is not None: - if jurisdiction.code not in known_search_results: - logger.warning( - "No known search results found for %s; skipping " - "jurisdiction", - jurisdiction.full_name, - ) - continue - else: - yield jurisdiction, known_search_results[jurisdiction.code] - else: + if known_search_results is None: yield jurisdiction, None + continue + + results = _known_source_for_code( + known_search_results, fips_to_str(jurisdiction.code) + ) + if results is None: + logger.warning( + "No known search results found for %s; skipping jurisdiction", + jurisdiction.full_name, + ) + continue + yield jurisdiction, results async def _load_known_se_results(runtime): @@ -713,7 +717,12 @@ async def _compute_total_cost(rate_tracker): def _known_source_for_code(sources, code): - """Prefer full codes, then accept keys without leading zeros""" + """Look up by jurisdiction code, tolerating dropped leading zeros + + Codes keep their leading zeros (``08001``), but configs and + manifests written before that stored them as numbers (``8001``). + The full code is tried first, then the same code without zeros. + """ if code in sources: return sources[code] if code.isdecimal(): diff --git a/compass/utilities/finalize.py b/compass/utilities/finalize.py index 9571283b9..4e6daf421 100644 --- a/compass/utilities/finalize.py +++ b/compass/utilities/finalize.py @@ -324,10 +324,12 @@ def _normalize_qualitative_rows(db): they share a single output file and the ``quantitative`` flag is not published. - Their ``summary`` is also copied into ``value``. The schemas ask the - LLM for a summary and a null value on these rows, so filling the - value here means the model only ever writes the requirement once, - while readers still find it in the column they expect. + Their ``summary`` is also copied into ``value`` where the value is + empty. The schemas ask the LLM for a summary and a null value on + most of these rows, so filling the value here means the model only + ever writes the requirement once, while readers still find it in + the column they expect. A value the model did supply (such as the + verbatim text of a definition) is kept. """ qualitative = ~db["quantitative"] db.loc[qualitative, "units"] = QUALITATIVE_UNITS @@ -335,7 +337,12 @@ def _normalize_qualitative_rows(db): # a numeric-only "value" column is float-typed, so it has to be # widened before qualitative text can be written into it db["value"] = db["value"].astype(object) - db.loc[qualitative, "value"] = db.loc[qualitative, "summary"] + empty = db["value"].isna() | ~db["value"].astype(str).str.strip().astype( + bool + ) + db.loc[qualitative & empty, "value"] = db.loc[ + qualitative & empty, "summary" + ] return db diff --git a/tests/python/unit/pipeline/test_pipeline_orchestration.py b/tests/python/unit/pipeline/test_pipeline_orchestration.py index d5a56d54e..5b15f1b5f 100644 --- a/tests/python/unit/pipeline/test_pipeline_orchestration.py +++ b/tests/python/unit/pipeline/test_pipeline_orchestration.py @@ -110,6 +110,49 @@ def test_known_sources_preserve_fips(tmp_path, from_file): assert runtime.known_doc_urls == known_doc_urls +@pytest.mark.parametrize( + ("stored", "code", "found"), + [ + ("08001", "08001", True), + ("8001", "08001", True), # written before codes kept their zeros + ("8013", "08001", False), + ("0648914", "0648914", True), + ], +) +def test_known_source_lookup_tolerates_dropped_leading_zeros( + stored, code, found +): + """Configs and manifests keyed by unpadded codes still match""" + sources = {stored: ["saved"]} + out = coordinator_module._known_source_for_code(sources, code) + assert (out == ["saved"]) is found + + +@pytest.mark.asyncio +async def test_saved_search_results_match_unpadded_codes(monkeypatch): + """A search manifest keyed by ``8001`` is replayed for ``08001``""" + + async def saved(_runtime): # ruff:ignore[unused-async] + return {"8001": ["https://example.com/a"]} + + monkeypatch.setattr(coordinator_module, "_load_known_se_results", saved) + monkeypatch.setattr( + coordinator_module, + "jurisdictions_from_df", + lambda _df: [ + SimpleNamespace(code="08001", full_name="Adams County, Colorado"), + SimpleNamespace(code="08013", full_name="Boulder County"), + ], + ) + matched = [ + (jurisdiction.code, results) + async for jurisdiction, results in ( + coordinator_module._jurisdictions_with_search_input(None, None) + ) + ] + assert matched == [("08001", ["https://example.com/a"])] + + def test_known_local_docs_missing_file(tmp_path): """Raise when known_local_docs points to missing config""" missing_fp = tmp_path / "does_not_exist.json" diff --git a/tests/python/unit/utilities/test_utilities_finalize.py b/tests/python/unit/utilities/test_utilities_finalize.py index 9ff6cd07d..45161c1f4 100644 --- a/tests/python/unit/utilities/test_utilities_finalize.py +++ b/tests/python/unit/utilities/test_utilities_finalize.py @@ -584,6 +584,36 @@ def _row(feature, quantitative, units): assert set(qual["feature"]) == {"Decommissioning", "Signage"} +def test_save_db_keeps_a_supplied_qualitative_value(tmp_path): + """A qualitative value the model supplied is not overwritten""" + out_cols = [col.name for col in COMPASSWindExtractor.OUTPUT_COLUMNS] + + def _row(feature, value, summary): + row = dict.fromkeys([*out_cols, "quantitative"]) + row.update( + feature=feature, + quantitative=False, + value=value, + summary=summary, + ordinance_text="quoted text", + ) + return row + + df = pd.DataFrame( + [ + _row("Glare", "Verbatim rule text.", "Paraphrase."), + _row("Color", " ", "Neutral colors."), + _row("Signage", None, "Warning signs only."), + ] + ) + finalize.save_db(df, tmp_path, COMPASSWindExtractor.OUTPUT_COLUMNS) + + out = pd.read_csv(tmp_path / "ordinances.csv").set_index("feature") + assert out.loc["Glare", "value"] == "Verbatim rule text." + assert out.loc["Color", "value"] == "Neutral colors." + assert out.loc["Signage", "value"] == "Warning signs only." + + def test_save_db_mirrors_summary_into_value_for_qualitative(tmp_path): """Copy summary into value on qualitative rows""" From d64818cb03650d00ccd35d93912056ff916117d7 Mon Sep 17 00:00:00 2001 From: Rajendra Adhikari Date: Thu, 12 Nov 2026 14:25:23 -0600 Subject: [PATCH 12/13] Let a plugin set its heuristic match threshold; one data center mention is enough --- .../data_centers/plugin_config.yaml | 14 +++----- compass/plugin/one_shot/base.py | 33 ++++++++++++++++++- .../test_extraction_data_centers_schema.py | 24 ++++++++++++++ .../unit/plugin/test_plugin_one_shot_base.py | 32 ++++++++++++++++++ 4 files changed, 93 insertions(+), 10 deletions(-) diff --git a/compass/extraction/data_centers/plugin_config.yaml b/compass/extraction/data_centers/plugin_config.yaml index d9bd0525f..e39d4da4b 100644 --- a/compass/extraction/data_centers/plugin_config.yaml +++ b/compass/extraction/data_centers/plugin_config.yaml @@ -44,6 +44,11 @@ website_keywords: planning: 180 heuristic_keywords: + # One mention of a data center term is enough for text to count, so + # every entry here must be specific to data centers. Generic zoning + # terms are left out: with a threshold of one they would let any + # zoning text through on their own. + min_matches: 1 GOOD_TECH_KEYWORDS: - datacenter - hyperscale @@ -56,18 +61,9 @@ heuristic_keywords: - computer server - server farm - colocation facility - - backup generator - - emergency generator - - zoning ordinance - - code of ordinances - - municipal code - - unified development ordinance - - conditional use permit GOOD_TECH_ACRONYMS: - PUE - WUE - - BESS - - UPS NOT_TECH_WORDS: - community center - recreation center diff --git a/compass/plugin/one_shot/base.py b/compass/plugin/one_shot/base.py index 66ae2938b..6aa434ad2 100644 --- a/compass/plugin/one_shot/base.py +++ b/compass/plugin/one_shot/base.py @@ -40,6 +40,8 @@ logger = logging.getLogger(__name__) _SCHEMA_DIR = importlib.resources.files("compass.plugin.one_shot.schemas") +_MIN_MATCHES = "MIN_MATCHES" +"""Heuristic keywords entry holding the plugin's match threshold""" _QT_SEMAPHORE = Semaphore(1) _WK_SEMAPHORE = Semaphore(1) _HK_SEMAPHORE = Semaphore(1) @@ -99,7 +101,11 @@ def create_schema_based_one_shot_extraction_plugin(config, tech): lists used by the heuristic document filter. The dictionary must include ``not_tech_words``, ``good_tech_keywords``, ``good_tech_acronyms``, and - ``good_tech_phrases`` keys. Alternatively, this input can + ``good_tech_phrases`` keys. It may also set + ``min_matches``, the number of distinct entries that must + appear for text to pass (default ``2``); use ``1`` only + when every entry is specific to the technology, so one + mention is enough. Alternatively, this input can simply be ``True``, in which case the LLM will be used to generate heuristic keyword lists based on the schema input. If ``False``, ``None``, or not provided, a `NoOp` @@ -291,6 +297,9 @@ class SchemaBasedHeuristic(KeywordBasedHeuristic): GOOD_TECH_KEYWORDS = hk["GOOD_TECH_KEYWORDS"] GOOD_TECH_ACRONYMS = hk["GOOD_TECH_ACRONYMS"] GOOD_TECH_PHRASES = hk["GOOD_TECH_PHRASES"] + MIN_DEFAULT_MATCHES = hk.get( + _MIN_MATCHES, KeywordBasedHeuristic.MIN_DEFAULT_MATCHES + ) self.__class__.HEURISTIC_KEYWORDS = hk self.__class__.HEURISTIC = SchemaBasedHeuristic @@ -614,6 +623,7 @@ def _normalize_heuristic_keywords(raw): "GOOD_TECH_PHRASES", } + raw, min_matches = _split_min_matches(raw) normalized = _normalize_input_kw(raw, expected_keys) _verify_expected_kw_are_not_missing(normalized, expected_keys) @@ -621,9 +631,30 @@ def _normalize_heuristic_keywords(raw): _verify_min_number_of_kw_provided(normalized) _warn_if_not_enough_kw_provided(normalized) + if min_matches is not None: + normalized[_MIN_MATCHES] = min_matches return normalized +def _split_min_matches(raw): + """Separate the optional match threshold from the keyword lists""" + lists, min_matches = {}, None + for key, value in raw.items(): + if isinstance(key, str) and key.strip().upper() == _MIN_MATCHES: + min_matches = value + else: + lists[key] = value + if min_matches is None: + return lists, None + if isinstance(min_matches, bool) or not isinstance(min_matches, int): + msg = "Heuristic `min_matches` must be a positive integer." + raise COMPASSPluginConfigurationError(msg) + if min_matches < 1: + msg = "Heuristic `min_matches` must be a positive integer." + raise COMPASSPluginConfigurationError(msg) + return lists, min_matches + + def _normalize_input_kw(raw, expected_keys): """Normalize the input keyword dictionary""" normalized = {} diff --git a/tests/python/unit/extraction/test_extraction_data_centers_schema.py b/tests/python/unit/extraction/test_extraction_data_centers_schema.py index 66711b3b2..75c42125d 100644 --- a/tests/python/unit/extraction/test_extraction_data_centers_schema.py +++ b/tests/python/unit/extraction/test_extraction_data_centers_schema.py @@ -3,7 +3,10 @@ import json from pathlib import Path +import pytest + import compass +from compass.extraction.data_centers import COMPASSDataCentersExtractor SCHEMA_FP = ( @@ -99,3 +102,24 @@ def test_units_use_canonical_vocabulary_without_detail(): assert "exactly as written" not in units_rule, name if name.endswith("districts"): assert units_rule.strip() == "null.", name + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("text", "passes"), + [ + ("Data centers shall maintain a 100-foot setback.", True), + ("A hyperscale facility requires a special exception.", True), + ("The zoning ordinance requires a conditional use permit.", False), + ("A backup generator is required for warehouses.", False), + ("UPS trucks deliver daily under the municipal code.", False), + ("The community center may host a data fair.", False), + ], +) +async def test_one_data_center_mention_passes_the_heuristic(text, passes): + """One specific term is enough; generic zoning terms never are""" + extractor = COMPASSDataCentersExtractor.__new__( + COMPASSDataCentersExtractor + ) + heuristic = await COMPASSDataCentersExtractor.get_heuristic(extractor) + assert heuristic.check(text) is passes diff --git a/tests/python/unit/plugin/test_plugin_one_shot_base.py b/tests/python/unit/plugin/test_plugin_one_shot_base.py index 161a73086..7255a7e28 100644 --- a/tests/python/unit/plugin/test_plugin_one_shot_base.py +++ b/tests/python/unit/plugin/test_plugin_one_shot_base.py @@ -8,6 +8,7 @@ import pandas as pd import pyjson5 +import compass.plugin.one_shot.base as one_shot_base from compass.plugin.one_shot.base import ( _out_cols_from_config, create_schema_based_one_shot_extraction_plugin, @@ -233,5 +234,36 @@ async def test_one_shot_plugin_loads_tiered_website_keywords(tmp_path): } +_KEYWORD_LISTS = { + "not_tech_words": ["community center"], + "good_tech_keywords": ["datacenter", "hyperscale"], + "good_tech_acronyms": ["PUE"], + "good_tech_phrases": ["data center", "server farm"], +} + + +def test_heuristic_keywords_accept_a_match_threshold(): + """A plugin may set how many distinct entries text must match""" + with pytest.warns(COMPASSPluginConfigurationWarning): + out = one_shot_base._normalize_heuristic_keywords( + {**_KEYWORD_LISTS, "min_matches": 1} + ) + default = one_shot_base._normalize_heuristic_keywords(_KEYWORD_LISTS) + assert out["MIN_MATCHES"] == 1 + assert out["GOOD_TECH_PHRASES"] == default["GOOD_TECH_PHRASES"] + assert "MIN_MATCHES" not in default + + +@pytest.mark.parametrize("value", [0, -1, 1.5, "2", True]) +def test_heuristic_match_threshold_must_be_a_positive_integer(value): + """An unusable threshold is rejected when the plugin is loaded""" + with pytest.raises( + one_shot_base.COMPASSPluginConfigurationError, match="min_matches" + ): + one_shot_base._normalize_heuristic_keywords( + {**_KEYWORD_LISTS, "min_matches": value} + ) + + if __name__ == "__main__": pytest.main(["-q", "--show-capture=all", Path(__file__), "-rapP"]) From e6bd7edee57f8ff4d5fc58ae47c5d4210c2601e9 Mon Sep 17 00:00:00 2001 From: Rajendra Adhikari Date: Thu, 12 Nov 2026 17:39:36 -0600 Subject: [PATCH 13/13] Use main's jurisdiction code handling instead of preserving leading zeros --- compass/pipeline/coordinator.py | 53 +++------ compass/pipeline/runtime.py | 2 +- compass/utilities/jurisdictions.py | 39 +------ .../test_integrated_pipeline_orchestration.py | 9 +- ...test_integrated_utilities_jurisdictions.py | 23 ---- .../pipeline/test_pipeline_orchestration.py | 104 ------------------ .../utilities/test_utilities_jurisdictions.py | 97 ---------------- 7 files changed, 26 insertions(+), 301 deletions(-) diff --git a/compass/pipeline/coordinator.py b/compass/pipeline/coordinator.py index 5cf8cd907..9319a294b 100644 --- a/compass/pipeline/coordinator.py +++ b/compass/pipeline/coordinator.py @@ -21,7 +21,7 @@ from compass.services.threaded import UsageUpdater from compass.services.threaded import GenericFuncRunner from compass.utilities.enums import COMPASSRunMode -from compass.utilities.jurisdictions import jurisdictions_from_df, fips_to_str +from compass.utilities.jurisdictions import jurisdictions_from_df from compass.utilities.logs import log_versions from compass.utilities.parsing import convert_paths_to_strings from compass.pipeline.collection.persistence import ( @@ -155,18 +155,15 @@ def _create( usage_tracker=usage_tracker, rate_tracker=self.runtime.rate_tracker, ) - source_key = fips_to_str(jurisdiction.code) return SingleJurisdictionRun( self.runtime, jurisdiction, extractor, usage_tracker=usage_tracker, - known_local_docs=_known_source_for_code( - self.runtime.known_local_docs, source_key - ), - known_doc_urls=_known_source_for_code( - self.runtime.known_doc_urls, source_key + known_local_docs=self.runtime.known_local_docs.get( + jurisdiction.code ), + known_doc_urls=self.runtime.known_doc_urls.get(jurisdiction.code), known_se_results=known_se_results, perform_se_search=self.runtime.request.perform_se_search, perform_website_search=( @@ -492,9 +489,7 @@ async def run(self, jurisdictions_df): tasks = [] start_date = datetime.now(UTC) for jurisdiction in jurisdictions_from_df(jurisdictions_df): - collection_info = _known_source_for_code( - collection_infos_by_fips, fips_to_str(jurisdiction.code) - ) + collection_info = collection_infos_by_fips.get(jurisdiction.code) if collection_info is None: logger.warning( "No collection info found for %s; skipping extraction", @@ -624,20 +619,18 @@ async def _jurisdictions_with_search_input(runtime, jurisdictions_df): known_search_results = await _load_known_se_results(runtime) for jurisdiction in jurisdictions_from_df(jurisdictions_df): - if known_search_results is None: + if known_search_results is not None: + if jurisdiction.code not in known_search_results: + logger.warning( + "No known search results found for %s; skipping " + "jurisdiction", + jurisdiction.full_name, + ) + continue + else: + yield jurisdiction, known_search_results[jurisdiction.code] + else: yield jurisdiction, None - continue - - results = _known_source_for_code( - known_search_results, fips_to_str(jurisdiction.code) - ) - if results is None: - logger.warning( - "No known search results found for %s; skipping jurisdiction", - jurisdiction.full_name, - ) - continue - yield jurisdiction, results async def _load_known_se_results(runtime): @@ -714,17 +707,3 @@ async def _compute_total_cost(rate_tracker): compute_total_cost_from_usage(total_usage), total_usage.get(rate_tracker.label), ) - - -def _known_source_for_code(sources, code): - """Look up by jurisdiction code, tolerating dropped leading zeros - - Codes keep their leading zeros (``08001``), but configs and - manifests written before that stored them as numbers (``8001``). - The full code is tried first, then the same code without zeros. - """ - if code in sources: - return sources[code] - if code.isdecimal(): - return sources.get(code.lstrip("0") or "0") - return None diff --git a/compass/pipeline/runtime.py b/compass/pipeline/runtime.py index dec78075e..312a6cec3 100644 --- a/compass/pipeline/runtime.py +++ b/compass/pipeline/runtime.py @@ -379,7 +379,7 @@ def _setup_folders( def _load_known_source(known_source): - """Load configured known sources with string jurisdiction codes""" + """Load configured known sources as int-keyed dictionaries""" known_source = known_source or {} if isinstance(known_source, str): known_source = load_config(known_source) diff --git a/compass/utilities/jurisdictions.py b/compass/utilities/jurisdictions.py index d6b3043ec..629e187df 100644 --- a/compass/utilities/jurisdictions.py +++ b/compass/utilities/jurisdictions.py @@ -326,9 +326,8 @@ def load_jurisdictions_from_fp(jurisdiction_fp): Parameters ---------- jurisdiction_fp : path-like - Path to a CSV with "FIPS" or jurisdiction name columns. - When supplied, FIPS is the sole matching key. Codes must match - the catalog, including leading zeros. + Path to csv file containing "County" and "State" columns that + define the jurisdictions for which info should be loaded. Returns ------- @@ -339,19 +338,16 @@ def load_jurisdictions_from_fp(jurisdiction_fp): Raises ------ COMPASSValueError - If a file without FIPS is missing required name columns - (``State`` or ``Jurisdiction Type`` for subdivisions). + If the input file is missing required columns (``State`` or + ``Jurisdiction Type`` when subdivisions are provided). Notes ----- Missing jurisdictions trigger warnings with a tabular summary. - A blank or unknown FIPS is skipped without falling back to names. """ jurisdictions = pd.read_csv(jurisdiction_fp, dtype=str).replace( {np.nan: None} ) - if "FIPS" in jurisdictions: - return _load_jurisdictions_by_fips(jurisdictions) jurisdictions = _validate_jurisdiction_input(jurisdictions) all_jurisdiction_info = load_all_jurisdiction_info() @@ -387,28 +383,6 @@ def load_jurisdictions_from_fp(jurisdiction_fp): return _format_jurisdiction_df_for_output(jurisdictions) -def _load_jurisdictions_by_fips(jurisdictions): - requested = jurisdictions[["FIPS"]].rename(columns={"FIPS": "FIPS_user"}) - requested["FIPS_user"] = requested["FIPS_user"].str.strip() - matched = requested.drop_duplicates().merge( - load_all_jurisdiction_info(), - left_on="FIPS_user", - right_on="FIPS", - how="left", - validate="one_to_one", - ) - matched = _filter_not_found_jurisdictions(matched, ["FIPS"]) - columns = [ - "County", - "State", - "Subdivision", - "Jurisdiction Type", - "FIPS", - "Website", - ] - return matched[columns].reset_index(drop=True) - - def jurisdictions_from_df(jurisdiction_info=None): """Convert rows DataFrame into Jurisdiction instances @@ -525,10 +499,7 @@ def _normalize_jurisdiction_name(name): def fips_to_str(value): - """[NOT PUBLIC API] Preserve strings when converting FIPS codes""" - if isinstance(value, str): - return value - + """[NOT PUBLIC API] Convert FIPS code to string""" with suppress(ValueError): value = int(value) return str(value) diff --git a/tests/python/integration/test_integrated_pipeline_orchestration.py b/tests/python/integration/test_integrated_pipeline_orchestration.py index d52665841..cbd447f66 100644 --- a/tests/python/integration/test_integrated_pipeline_orchestration.py +++ b/tests/python/integration/test_integrated_pipeline_orchestration.py @@ -733,7 +733,6 @@ def test_collection_cli_selectively_resets_and_collects_missing_shards( "County,State\nWhatcom,Washington\nAdams,Colorado\nBoulder,Colorado\n", encoding="utf-8", ) - # keys without the catalog's leading zero still find their documents for code in ["8001", "8013"]: config["known_local_docs"][code] = config["known_local_docs"]["53073"] collection_cli_config.write_text(json.dumps(config), encoding="utf-8") @@ -746,14 +745,14 @@ def test_collection_cli_selectively_resets_and_collects_missing_shards( for path in output.rglob("*_collection_manifest.json") } passing_bytes = shards["53073"].read_bytes() - failed = json.loads(shards["08001"].read_text(encoding="utf-8")) + failed = json.loads(shards["8001"].read_text(encoding="utf-8")) failed.update( num_docs=0, results=[], completed_step_document_counts={"known_local_docs": 0}, ) - shards["08001"].write_text(json.dumps(failed), encoding="utf-8") - shards["08013"].unlink() + shards["8001"].write_text(json.dumps(failed), encoding="utf-8") + shards["8013"].unlink() seed_fp = output / "search_input.json" seed_fp.write_text('{"preserved": true}', encoding="utf-8") COMPASS_PB.reset() @@ -765,7 +764,7 @@ def test_collection_cli_selectively_resets_and_collects_missing_shards( assert result.exit_code == 0, f"{result.output}\n{result.exception}" assert shards["53073"].read_bytes() == passing_bytes - for code in ["08001", "08013"]: + for code in ["8001", "8013"]: assert ( json.loads(shards[code].read_text(encoding="utf-8"))["num_docs"] == 1 diff --git a/tests/python/integration/test_integrated_utilities_jurisdictions.py b/tests/python/integration/test_integrated_utilities_jurisdictions.py index c06f337f5..2c2300669 100644 --- a/tests/python/integration/test_integrated_utilities_jurisdictions.py +++ b/tests/python/integration/test_integrated_utilities_jurisdictions.py @@ -50,29 +50,6 @@ def test_load_all_jurisdictions(): assert "Rhode Island" in set(jurisdiction_info["State"]) -def test_fips_matching_with_missing_catalog_county(tmp_path): - """Match actual catalog codes across county omissions and place types.""" - path = tmp_path / "jurisdictions.csv" - pd.DataFrame( - { - "FIPS": ["0107000", "1723256", "4204183800", "5183136"], - "County": ["Jefferson", "Cook", "Cumberland", "Fauquier"], - } - ).to_csv(path, index=False) - - result = load_jurisdictions_from_fp(path) - - assert result["FIPS"].tolist() == [ - "0107000", "1723256", "4204183800", "5183136" - ] - assert result["Jurisdiction Type"].tolist() == [ - "city", "village", "township", "town" - ] - assert result["County"].tolist() == [ - "Jefferson", None, "Cumberland", None - ] - - def test_jurisdiction_websites(): """Test the `jurisdiction_websites` function""" diff --git a/tests/python/unit/pipeline/test_pipeline_orchestration.py b/tests/python/unit/pipeline/test_pipeline_orchestration.py index 5b15f1b5f..6f2a87106 100644 --- a/tests/python/unit/pipeline/test_pipeline_orchestration.py +++ b/tests/python/unit/pipeline/test_pipeline_orchestration.py @@ -1,11 +1,9 @@ """Tests for compass.pipeline orchestration""" -import json import logging from itertools import product from pathlib import Path from types import SimpleNamespace -from unittest.mock import Mock import pandas as pd import pytest @@ -85,74 +83,6 @@ async def run(self, jurisdictions_df): return DummyWorkflow -@pytest.mark.parametrize("from_file", [False, True]) -def test_known_sources_preserve_fips(tmp_path, from_file): - """Keep leading zeros in local-document and URL lookup keys""" - known_local_docs = {"0648914": [{"source_fp": "ordinance.pdf"}]} - known_doc_urls = {"0648914": ["https://example.com/ordinance.pdf"]} - if from_file: - local_fp = tmp_path / "local.json" - local_fp.write_text(json.dumps(known_local_docs), encoding="utf-8") - urls_fp = tmp_path / "urls.json" - urls_fp.write_text(json.dumps(known_doc_urls), encoding="utf-8") - - request = CollectionRequest( - out_dir=tmp_path / "outputs", - tech="solar", - jurisdiction_fp=tmp_path / "jurisdictions.csv", - known_local_docs=str(local_fp) if from_file else known_local_docs, - known_doc_urls=str(urls_fp) if from_file else known_doc_urls, - ) - - runtime = PipelineRuntime(request) - - assert runtime.known_local_docs == known_local_docs - assert runtime.known_doc_urls == known_doc_urls - - -@pytest.mark.parametrize( - ("stored", "code", "found"), - [ - ("08001", "08001", True), - ("8001", "08001", True), # written before codes kept their zeros - ("8013", "08001", False), - ("0648914", "0648914", True), - ], -) -def test_known_source_lookup_tolerates_dropped_leading_zeros( - stored, code, found -): - """Configs and manifests keyed by unpadded codes still match""" - sources = {stored: ["saved"]} - out = coordinator_module._known_source_for_code(sources, code) - assert (out == ["saved"]) is found - - -@pytest.mark.asyncio -async def test_saved_search_results_match_unpadded_codes(monkeypatch): - """A search manifest keyed by ``8001`` is replayed for ``08001``""" - - async def saved(_runtime): # ruff:ignore[unused-async] - return {"8001": ["https://example.com/a"]} - - monkeypatch.setattr(coordinator_module, "_load_known_se_results", saved) - monkeypatch.setattr( - coordinator_module, - "jurisdictions_from_df", - lambda _df: [ - SimpleNamespace(code="08001", full_name="Adams County, Colorado"), - SimpleNamespace(code="08013", full_name="Boulder County"), - ], - ) - matched = [ - (jurisdiction.code, results) - async for jurisdiction, results in ( - coordinator_module._jurisdictions_with_search_input(None, None) - ) - ] - assert matched == [("08001", ["https://example.com/a"])] - - def test_known_local_docs_missing_file(tmp_path): """Raise when known_local_docs points to missing config""" missing_fp = tmp_path / "does_not_exist.json" @@ -170,40 +100,6 @@ def test_known_local_docs_missing_file(tmp_path): PipelineRuntime(request) -@pytest.mark.parametrize("use_canonical_key", [False, True]) -def test_known_sources_match_fips_with_leading_zero( - monkeypatch, use_canonical_key -): - """Canonical FIPS codes still find numerically keyed known sources.""" - runtime = SimpleNamespace( - extractor_class=Mock(), - models={}, - rate_tracker=None, - search_params=SimpleNamespace(num_search_results_to_crawl=0), - known_local_docs={"8041": ["ordinance.pdf"]}, - known_doc_urls={"8041": ["https://example.com/ordinance.pdf"]}, - request=SimpleNamespace( - perform_se_search=True, perform_website_search=False - ), - ) - if use_canonical_key: - runtime.known_local_docs["08041"] = ["canonical.pdf"] - runtime.known_doc_urls["08041"] = ["https://example.com/canonical.pdf"] - create_run = Mock() - monkeypatch.setattr( - coordinator_module, "SingleJurisdictionRun", create_run - ) - workflow = coordinator_module.COMPASSFullProcessing(runtime) - - workflow._create(SimpleNamespace(code="08041")) - - filename = "canonical.pdf" if use_canonical_key else "ordinance.pdf" - assert create_run.call_args.kwargs["known_local_docs"] == [filename] - assert create_run.call_args.kwargs["known_doc_urls"] == [ - f"https://example.com/{filename}" - ] - - def test_known_local_docs_logs_missing_file(tmp_path, testing_log_file): """Log missing known_local_docs config to error file""" diff --git a/tests/python/unit/utilities/test_utilities_jurisdictions.py b/tests/python/unit/utilities/test_utilities_jurisdictions.py index 4fb73a811..78f952273 100644 --- a/tests/python/unit/utilities/test_utilities_jurisdictions.py +++ b/tests/python/unit/utilities/test_utilities_jurisdictions.py @@ -7,7 +7,6 @@ import numpy as np from compass.utilities.jurisdictions import ( - fips_to_str, load_jurisdictions_from_fp, load_jurisdictions_from_subdivision_names, jurisdiction_websites, @@ -124,102 +123,6 @@ def test_load_jurisdictions_from_fp(tmp_path): assert {type(val) for val in jurisdictions["FIPS"]} == {str} -def test_load_jurisdictions_by_fips_ignores_names(tmp_path): - """FIPS overrides conflicting names and deduplicates requests.""" - path = tmp_path / "jurisdictions.csv" - pd.DataFrame( - { - "FIPS": ["08041", " 08041 ", "49003"], - "State": ["Wrong state"] * 3, - "County": ["Wrong county"] * 3, - } - ).to_csv(path, index=False) - - result = load_jurisdictions_from_fp(path) - - assert result["FIPS"].tolist() == ["08041", "49003"] - assert result["State"].tolist() == ["Colorado", "Utah"] - assert result["County"].tolist() == ["El Paso", "Box Elder"] - - -def test_load_jurisdictions_from_fips_only(tmp_path): - """A FIPS-only file preserves leading zeros without requiring names.""" - path = tmp_path / "jurisdictions.csv" - pd.DataFrame({"FIPS": ["08041"]}).to_csv(path, index=False) - - result = load_jurisdictions_from_fp(path) - - assert result["FIPS"].tolist() == ["08041"] - assert result["Jurisdiction Type"].tolist() == ["county"] - - -def test_load_jurisdictions_invalid_fips_do_not_fall_back_to_names(tmp_path): - """Unknown and missing codes cannot select a different named location.""" - path = tmp_path / "jurisdictions.csv" - pd.DataFrame( - { - "FIPS": ["9999999999", None, "08041"], - "State": ["Colorado"] * 3, - "County": ["El Paso"] * 3, - } - ).to_csv(path, index=False) - - with pytest.warns(COMPASSWarning, match="9999999999"): - result = load_jurisdictions_from_fp(path) - - assert result["FIPS"].tolist() == ["08041"] - - -@pytest.mark.parametrize( - ("state", "county", "subdivision", "jurisdiction_type", "fips"), - [ - ("California", None, "Monterey Park", "city", "0648914"), - ("Colorado", "Adams", None, "county", "08001"), - ("California", None, None, "state", "06000"), - ("Alabama", "Autauga", "Marbury", "city", "0100192106"), - ], -) -def test_load_jurisdictions_from_fp_preserves_fips( - tmp_path, state, county, subdivision, jurisdiction_type, fips -): - """Preserve canonical codes through CSV lookup and object creation""" - jurisdiction_fp = tmp_path / "jurisdictions.csv" - pd.DataFrame( - { - "State": [state], - "County": [county], - "Subdivision": [subdivision], - "Jurisdiction Type": [jurisdiction_type], - } - ).to_csv(jurisdiction_fp, index=False) - - jurisdictions = load_jurisdictions_from_fp(jurisdiction_fp) - - assert jurisdictions["FIPS"].tolist() == [fips] - (jurisdiction,) = jurisdictions_from_df(jurisdictions) - assert jurisdiction.code == fips - assert jurisdiction.website_url == jurisdiction_websites()[fips] - - -@pytest.mark.parametrize( - ("value", "expected"), - [ - ("0648914", "0648914"), - ("08001", "08001"), - ("06", "06"), - ("0100192106", "0100192106"), - ("district-01", "district-01"), - (18031, "18031"), - (18031.0, "18031"), - (np.int64(18031), "18031"), - (np.float64(18031), "18031"), - ], -) -def test_fips_to_str(value, expected): - """Preserve string identifiers and support numeric codes""" - assert fips_to_str(value) == expected - - def test_load_jurisdictions_from_fp_bad_input(tmp_path): """Test `load_jurisdictions_from_fp` function"""