openapi json export

.github/workflows/docs.yml

@@ -15,7 +15,9 @@ jobs:
       - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@v2
         with:
-          version: '1.10'
+          # `[sources]` is honored from 1.11 on; on 1.10 the docs env silently ignored the
+          # OpenAPI pins and fell through to the registry.
+          version: '1.12'
       - name: Install dependencies
         run: julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
       - name: Build and deploy

.gitignore

@@ -13,6 +13,9 @@ generated*.md
 data
 *sqlite
 
+# scripts/inspect_14bus_json.jl output — regenerable dev-inspection JSON, not a fixture
+inspection_output/
+
 # System-specific files and directories generated by the BinaryProvider and BinDeps packages
 # They contain absolute paths specific to the host computer, and so should not be committed
 deps/deps.jl
@@ -124,3 +127,9 @@ $RECYCLE.BIN/
 
 ## Acknowledgements
 # Many thanks to `https://gitignore.io/`, written and maintained by Joe Blau, which contributed much material to this gitignore file.
+
+# Private ERCOT SSWG cases — never commit
+SSWG/
+
+# Claude working plans — never commit
+.claude/plans/

Project.toml

@@ -7,15 +7,27 @@ authors = ["Sienna Team"]
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
 InfrastructureSystems = "2cd47ed4-ca9b-11e9-27f2-ab636a7671f1"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+OpenAPI = "d5e62ea6-ddf3-4d43-8e4c-ad5e6c8bfd7d"
+PowerCoreOpenAPIModels = "b7b40286-e793-417d-a9a0-b1583e4da1cb"
+PowerOperationsOpenAPIModels = "a372b6d7-45a2-44c2-8199-6a724b72e8ff"
 Unicode = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
-YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6"
+
+[sources]
+# These packages are generated and not yet released, so they resolve from the branch
+# rather than the registry. Switch to a tagged rev once one exists.
+PowerCoreOpenAPIModels = {url = "https://github.com/Sienna-Platform/PowerOpenAPIModels.git", rev = "jd/openapi_regen", subdir = "PowerCoreOpenAPIModels.jl"}
+PowerOperationsOpenAPIModels = {url = "https://github.com/Sienna-Platform/PowerOpenAPIModels.git", rev = "jd/openapi_regen", subdir = "PowerOperationsOpenAPIModels.jl"}
 
 [compat]
 DataStructures = "0.19.3"
 DocStringExtensions = "~0.8, ~0.9"
 InfrastructureSystems = "^3.2"
+JSON = "1"
 LinearAlgebra = "1"
+OpenAPI = "0.2"
+PowerCoreOpenAPIModels = "0.1"
+PowerOperationsOpenAPIModels = "0.1"
 Unicode = "1"
-YAML = "0.4.16"
 julia = "^1.10"

docs/Project.toml

@@ -2,7 +2,17 @@
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 DocumenterInterLinks = "d12716ef-a0f6-4df4-a9f1-a5a34e75c656"
+PowerCoreOpenAPIModels = "b7b40286-e793-417d-a9a0-b1583e4da1cb"
 PowerFlowFileParser = "bed98974-b02e-5e2f-9ee0-a103f5c450dd"
+PowerOperationsOpenAPIModels = "a372b6d7-45a2-44c2-8199-6a724b72e8ff"
+
+# The two OpenAPI packages are transitive through PowerFlowFileParser, but `[sources]` is
+# honored only for the active project's own deps, so the docs env has to declare them to
+# pin them.
+[sources]
+PowerFlowFileParser = {path = ".."}
+PowerCoreOpenAPIModels = {url = "https://github.com/Sienna-Platform/PowerOpenAPIModels.git", rev = "jd/openapi_regen", subdir = "PowerCoreOpenAPIModels.jl"}
+PowerOperationsOpenAPIModels = {url = "https://github.com/Sienna-Platform/PowerOpenAPIModels.git", rev = "jd/openapi_regen", subdir = "PowerOperationsOpenAPIModels.jl"}
 
 [compat]
 julia = "^1.10"

scripts/README.md

@@ -0,0 +1,30 @@
+# scripts
+
+## `validate_with_python.py`
+
+Validates a JSON document written by `to_json` against the pydantic models generated from
+the same SiennaSchemas, catching disagreements between the two generators.
+
+```bash
+python3 -m pip install -e ../power-openapi-models   # once, from a sibling checkout
+python3 scripts/validate_with_python.py case.json
+```
+
+If your system Python refuses the install with `error: externally-managed-environment`
+(PEP 668), install into a virtual environment instead and run the script with that
+venv's interpreter:
+
+```bash
+python3 -m venv /path/to/venv
+/path/to/venv/bin/python3 -m pip install -e ../power-openapi-models
+/path/to/venv/bin/python3 scripts/validate_with_python.py case.json
+```
+
+Run by hand. It is not part of `julia --project=test test/runtests.jl`, which stays free of
+a Python dependency.
+
+## `formatter/`
+
+```bash
+julia --project=scripts/formatter -e 'include("scripts/formatter/formatter_code.jl")'
+```

scripts/inspect_14bus_json.jl

@@ -0,0 +1,135 @@
+#!/usr/bin/env julia
+#
+# Parse a PSS/E case and write JSON for visual inspection.
+#
+#   julia --project scripts/inspect_14bus_json.jl
+#   julia --project scripts/inspect_14bus_json.jl --case path/to/other.raw --out /tmp/look
+#
+# Produces three files in the output directory (default `inspection_output/`):
+#
+#   <case>.pm.json                  the parsed PowerModels dict
+#   <case>.NATURAL_UNITS.json       OpenAPI document, unit_system = NATURAL_UNITS
+#   <case>.DEVICE_BASE.json         OpenAPI document, unit_system = DEVICE_BASE
+#
+# Both OpenAPI documents are built via `build_openapi_system` and carry real components —
+# buses, loads, generators, branches, transformers, dc lines, shunts, plus
+# `ImpedanceCorrectionData` and `DiscreteControlledACBranch`.
+# `../power-openapi-models/scripts/check_json_compat.py` reads this directory's `*.json`
+# (excluding `.pm.json`/`.roundtrip.json`) to validate Julia's output against the
+# generated Python models.
+
+import JSON
+using PowerFlowFileParser
+
+const DEFAULT_CASE = joinpath(@__DIR__, "..", "test", "modified_14bus_system.raw")
+const UNIT_SYSTEMS = ("NATURAL_UNITS", "DEVICE_BASE")
+
+function parse_args(argv)
+    case = DEFAULT_CASE
+    out = joinpath(@__DIR__, "..", "inspection_output")
+    i = 1
+    while i <= length(argv)
+        arg = argv[i]
+        if arg == "--case"
+            case = argv[i + 1]
+            i += 2
+        elseif arg == "--out"
+            out = argv[i + 1]
+            i += 2
+        elseif arg in ("-h", "--help")
+            println(read(@__FILE__, String)[1:findfirst("import JSON", read(@__FILE__, String)).start - 1])
+            exit(0)
+        else
+            error("Unrecognized argument: $arg")
+        end
+    end
+    return (case = abspath(case), out = abspath(out))
+end
+
+"""Write `data` as indented JSON, reporting the path and size."""
+function write_json(path, data)
+    open(path, "w") do io
+        JSON.print(io, data, 2)
+    end
+    println("  wrote $(relpath(path)) ($(round(filesize(path) / 1024; digits = 1)) KiB)")
+    return path
+end
+
+"""Count entries per top-level PM-dict section, so the parse can be eyeballed."""
+function summarize_pm(data)
+    println("\nParsed sections (component counts):")
+    for key in sort(collect(keys(data)))
+        value = data[key]
+        if value isa Dict && !isempty(value) && all(v -> v isa Dict, values(value))
+            println("  $(rpad(key, 26)) $(length(value))")
+        elseif value isa Vector
+            println("  $(rpad(key, 26)) $(length(value))")
+        end
+    end
+    println("\nScalars: base_power=$(get(data, "baseMVA", "?")) per_unit=$(get(data, "per_unit", "?"))")
+    return nothing
+end
+
+"""
+Report the shunts, including any relocated off a switching device.
+
+A BRANCH row with a '@'/'*' CKT is a breaker or switch and has no line shunt, so
+GI/BI/GJ/BJ on such a row is malformed input and gets moved to the bus it was
+declared on. Those carry a `branch shunt` source_id, so they are worth calling out
+separately during a visual check.
+"""
+function summarize_shunts(data)
+    shunts = get(data, "shunt", Dict())
+    if isempty(shunts)
+        println("\nNo shunts in this case.")
+        return nothing
+    end
+    println("\nShunts ($(length(shunts))):")
+    for key in sort(collect(keys(shunts)))
+        s = shunts[key]
+        source = get(s, "source_id", Any[])
+        tag = isempty(source) ? "?" : string(first(source))
+        marker = tag == "branch shunt" ? "  <-- relocated from a switching device" : ""
+        println(
+            "  bus=$(rpad(get(s, "shunt_bus", "?"), 6)) " *
+            "gs=$(rpad(get(s, "gs", "?"), 12)) bs=$(rpad(get(s, "bs", "?"), 12)) " *
+            "source=$tag$marker",
+        )
+    end
+    return nothing
+end
+
+function main(argv)
+    opts = parse_args(argv)
+    isfile(opts.case) || error("Case not found: $(opts.case)")
+    mkpath(opts.out)
+    stem = first(splitext(basename(opts.case)))
+
+    println("Case:   $(opts.case)")
+    println("Output: $(opts.out)\n")
+
+    println("Parsing...")
+    pm = PowerModelsData(opts.case)
+    data = pm.data
+
+    println("\nWriting JSON:")
+    write_json(joinpath(opts.out, "$stem.pm.json"), data)
+
+    # The OpenAPI envelope in both unit systems, built from the same parsed case via the
+    # full emit layer.
+    for unit_system in UNIT_SYSTEMS
+        sys = build_openapi_system(pm; unit_system = unit_system)
+        path = joinpath(opts.out, "$stem.$unit_system.json")
+        to_json(sys, path; force = true, pretty = true)
+        println("  wrote $(relpath(path)) ($(round(filesize(path) / 1024; digits = 1)) KiB)")
+        println(
+            "    components: $(join(PowerFlowFileParser.component_type_names(sys), ", "))",
+        )
+    end
+
+    summarize_pm(data)
+    summarize_shunts(data)
+    return nothing
+end
+
+main(ARGS)

scripts/validate_with_python.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+"""Validate a Sienna OpenAPI JSON document against the generated pydantic models.
+
+The Julia and Python model packages are generated from the same SiennaSchemas, so a
+component that loads in one and not the other is a schema-level defect. Validates
+components, supplemental attributes, and both association sections (supplemental attribute
+and time series). Run this by hand against emitted documents; it is deliberately not wired
+into the Julia test suite, which stays free of a Python dependency.
+
+Usage:
+    python3 scripts/validate_with_python.py DOCUMENT.json
+"""
+
+import argparse
+import importlib
+import json
+import sys
+
+MODULES = (
+    "power_openapi_models.core.models",
+    "power_openapi_models.operations.models",
+    "power_openapi_models.investments.models",
+    "power_openapi_models.dynamics.models",
+)
+
+ASSOCIATION_SECTIONS = (
+    ("supplemental_attribute_associations", "SupplementalAttributeAssociation"),
+    ("time_series_associations", "TimeSeriesAssociation"),
+)
+
+
+def load_models():
+    """Map every pydantic class name to its class, first module winning."""
+    models = {}
+    for name in MODULES:
+        module = importlib.import_module(name)
+        for attr in dir(module):
+            if attr.startswith("_") or attr in models:
+                continue
+            models[attr] = getattr(module, attr)
+    return models
+
+
+def validate_group(models, type_name, payloads, failures):
+    model = models.get(type_name)
+    if model is None:
+        failures.append(f"{type_name}: no pydantic class of that name in {MODULES}")
+        return 0
+    validated = 0
+    for index, payload in enumerate(payloads):
+        try:
+            model.model_validate(payload)
+            validated += 1
+        except Exception as error:
+            failures.append(f"{type_name}[{index}]: {error}")
+    return validated
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("document", help="path to a JSON document written by to_json")
+    args = parser.parse_args()
+
+    with open(args.document) as handle:
+        document = json.load(handle)
+
+    models = load_models()
+    failures = []
+    validated = 0
+
+    for type_name in sorted(document.get("components", {})):
+        validated += validate_group(
+            models, type_name, document["components"][type_name], failures
+        )
+
+    # `supplemental_attributes` is a flat list carrying no type tag of its own; the
+    # association table is what names each attribute's class, by `attribute_id`. An
+    # attribute with no association has no declarable type, which is a document defect
+    # rather than something to validate against a guess.
+    attribute_types = {
+        assoc["attribute_id"]: assoc["attribute_type"]
+        for assoc in document.get("supplemental_attribute_associations", [])
+        if "attribute_id" in assoc and "attribute_type" in assoc
+    }
+    for index, payload in enumerate(document.get("supplemental_attributes", [])):
+        type_name = attribute_types.get(payload.get("id"))
+        if type_name is None:
+            failures.append(
+                f"supplemental_attributes[{index}]: id={payload.get('id')} has no "
+                "entry in supplemental_attribute_associations, so its type is unknown"
+            )
+            continue
+        validated += validate_group(models, type_name, [payload], failures)
+
+    for section, type_name in ASSOCIATION_SECTIONS:
+        validated += validate_group(
+            models, type_name, document.get(section, []), failures
+        )
+
+    print(f"validated {validated} components")
+    if failures:
+        print(f"{len(failures)} failure(s):", file=sys.stderr)
+        for failure in failures:
+            print(f"  {failure}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())