Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 76 additions & 22 deletions src/eopf_geozarr/conversion/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,16 @@
from __future__ import annotations

from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Protocol, TypedDict, TypeGuard, cast, runtime_checkable
from typing import (
TYPE_CHECKING,
Any,
Literal,
Protocol,
TypedDict,
TypeGuard,
cast,
runtime_checkable,
)

import numpy as np
import structlog
Expand Down Expand Up @@ -78,14 +87,7 @@ def _bbox_from_coordinates(dataset: xr.Dataset) -> list[float]:
along an axis, because the pixel size is then unknown.
"""
if not _has_regular_xy_grid(dataset):
x_coords = dataset.coords["x"].values
y_coords = dataset.coords["y"].values
return [
float(x_coords.min()),
float(y_coords.min()),
float(x_coords.max()),
float(y_coords.max()),
]
return _coordinate_extent(dataset)

pixel_size_x, _, x_min, _, neg_pixel_size_y, y_max = _transform_from_coordinates(dataset)
x_max = x_min + dataset.sizes["x"] * pixel_size_x
Expand Down Expand Up @@ -378,10 +380,36 @@ def get_subroots(groups: tuple[str, ...]) -> set[str] | None:
return subroots or None


def _node_transform_from_coordinates(dataset: xr.Dataset) -> GeoTransform:
"""Affine transform from array indices to the x/y node coordinates.

For a node-registered grid, index `(0, 0)` is at `(x[0], y[0])`: the coordinates
are not moved by a half cell. Requires `_has_regular_xy_grid(dataset)`.
"""
x_coords = dataset.coords["x"].values
y_coords = dataset.coords["y"].values
step_x = float(x_coords[1] - x_coords[0])
step_y = float(y_coords[1] - y_coords[0])
return (step_x, 0.0, float(x_coords[0]), 0.0, step_y, float(y_coords[0]))


def _coordinate_extent(dataset: xr.Dataset) -> list[float]:
"""Extent ``[xmin, ymin, xmax, ymax]`` of the x/y coordinate values."""
x_coords = dataset.coords["x"].values
y_coords = dataset.coords["y"].values
return [
float(x_coords.min()),
float(y_coords.min()),
float(x_coords.max()),
float(y_coords.max()),
]


def write_dataset_geo_metadata(
dataset: xr.Dataset,
grid_mapping_var_name: str = "spatial_ref",
crs: CRS | None = None,
registration: Literal["pixel", "node"] = "pixel",
) -> xr.Dataset:
"""
Return a copy of the dataset with geographic metadata added; `dataset` is not modified.
Expand All @@ -391,6 +419,11 @@ def write_dataset_geo_metadata(
dataset: Dataset to add metadata to
grid_mapping_var_name: Name for grid mapping variable
crs: Coordinate Reference System to use (if None, attempts to detect from dataset)
registration: `spatial:registration` of the grid. With `"pixel"`, the x/y
coordinates are cell centres, and `spatial:bbox` and `spatial:transform`
refer to the outer cell edges. With `"node"`, the values are at the x/y
coordinates, and `spatial:bbox` and `spatial:transform` refer to these
coordinates (e.g. the Sentinel-2 sun and viewing angle grids).

Returns:
The dataset with the metadata, or an unchanged copy if no CRS is found.
Expand All @@ -416,7 +449,14 @@ def write_dataset_geo_metadata(
dataset.rio.write_grid_mapping(grid_mapping_var_name, inplace=True)
dataset.attrs["grid_mapping"] = grid_mapping_var_name

has_xy_dims = "x" in dataset.dims and "y" in dataset.dims
for var in dataset.data_vars.values():
# A variable without the x/y dimensions (e.g. a mean angle per band)
# is not on the grid, so it gets no grid mapping.
if has_xy_dims and not {"x", "y"} <= set(var.dims):
var.attrs.pop("grid_mapping", None)
var.encoding.pop("grid_mapping", None)
continue
var.rio.write_grid_mapping(grid_mapping_var_name, inplace=True)
var.attrs["grid_mapping"] = grid_mapping_var_name

Expand All @@ -427,25 +467,36 @@ def write_dataset_geo_metadata(
# Assemble spatial convention data
spatial_data: spatial_cm.SpatialAttrs = {
"spatial:dimensions": ["y", "x"], # Required field
"spatial:registration": "pixel", # Default registration type
"spatial:registration": registration,
}

# Calculate and add spatial bbox if coordinates are available
if "x" in dataset.coords and "y" in dataset.coords:
spatial_data["spatial:bbox"] = _bbox_from_coordinates(dataset)
if registration == "node":
spatial_data["spatial:bbox"] = _coordinate_extent(dataset)
if _has_regular_xy_grid(dataset):
spatial_data["spatial:transform"] = list(
_node_transform_from_coordinates(dataset)
)
else:
spatial_data["spatial:bbox"] = _bbox_from_coordinates(dataset)

if _has_spatial_transform(dataset):
spatial_transform = _preferred_spatial_transform(dataset)
# Only add spatial:transform if we have valid transform data (not all zeros)
if not all(t == 0 for t in spatial_transform):
spatial_data["spatial:transform"] = list(spatial_transform)
if _has_spatial_transform(dataset):
spatial_transform = _preferred_spatial_transform(dataset)
# Only add spatial:transform if we have valid transform data (not all zeros)
if not all(t == 0 for t in spatial_transform):
spatial_data["spatial:transform"] = list(spatial_transform)

# Add spatial shape if data variables exist
# Add spatial shape if data variables exist. Take it from the x/y
# dimensions: the first variable is not always on the grid.
if dataset.data_vars:
first_var = next(iter(dataset.data_vars.values()))
if first_var.ndim >= 2:
height, width = first_var.shape[-2:]
spatial_data["spatial:shape"] = [height, width]
if has_xy_dims:
spatial_data["spatial:shape"] = [dataset.sizes["y"], dataset.sizes["x"]]
else:
first_var = next(iter(dataset.data_vars.values()))
if first_var.ndim >= 2:
height, width = first_var.shape[-2:]
spatial_data["spatial:shape"] = [height, width]

# Build validated spatial + proj convention attrs (data + CMOs) via zarr-cm
dataset.attrs.update(build_convention_attrs(spatial=spatial_data, crs=crs))
Expand All @@ -462,6 +513,7 @@ def stream_write_dataset(
add_geo_metadata: bool = False,
chunk_and_shard_coords: bool = True,
crs: CRS | None = None,
registration: Literal["pixel", "node"] = "pixel",
) -> xr.Dataset:
"""
Stream write a lazy dataset with advanced chunking and sharding.
Expand All @@ -474,6 +526,8 @@ def stream_write_dataset(
dataset_path: Output path for dataset
encoding: Encoding dictionary for variables
enable_sharding: Enable Zarr v3 sharding
registration: `spatial:registration` of the grid, used with
`add_geo_metadata` (see `write_dataset_geo_metadata`)

Returns:
Written dataset
Expand Down Expand Up @@ -504,7 +558,7 @@ def stream_write_dataset(
)

if add_geo_metadata:
dataset = write_dataset_geo_metadata(dataset, crs=crs)
dataset = write_dataset_geo_metadata(dataset, crs=crs, registration=registration)

# Sanitize NaN values in dataset attributes before writing
dataset = fs_utils.sanitize_dataset_attributes(dataset)
Expand Down
10 changes: 9 additions & 1 deletion src/eopf_geozarr/s2_optimization/s2_multiscale.py
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,9 @@ def applies_to(self, s2_type: S2Type | None) -> bool:
),
)

GEOMETRY_PATH = "/conditions/geometry"
"""Group of the sun and viewing angle grids (node-registered, in the tile CRS)."""

MultiscalesFlavor = Literal["experimental_multiscales_convention"]


Expand Down Expand Up @@ -681,7 +684,11 @@ def create_multiscale_from_datatree(
for name in scalar_coords:
encoding.pop(str(name), None)

add_geo_metadata = any(
# The sun and viewing angle grids in `/conditions/geometry` are in the tile CRS.
# Their values are at the grid nodes: `(x[0], y[0])` is the upper-left corner
# of the tile, not the centre of a cell.
is_geometry_group = group_path == GEOMETRY_PATH
add_geo_metadata = is_geometry_group or any(
m in group_path for m in ("/measurements/", "/quality/", "/conditions/mask")
)
ds_out = utils.stream_write_dataset(
Expand All @@ -691,6 +698,7 @@ def create_multiscale_from_datatree(
encoding=encoding,
add_geo_metadata=add_geo_metadata,
crs=crs,
registration="node" if is_geometry_group else "pixel",
)

processed_groups[group_path] = ds_out
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,49 @@
"attributes": {},
"members": {
"geometry": {
"attributes": {},
"attributes": {
"grid_mapping": "spatial_ref",
"proj:code": "EPSG:32632",
"spatial:bbox": [
302386.95652173914,
4892626.956521739,
407413.04347826086,
4997653.043478261
],
"spatial:dimensions": [
"y",
"x"
],
"spatial:registration": "node",
"spatial:shape": [
23,
23
],
"spatial:transform": [
4773.913043478271,
0.0,
302386.95652173914,
0.0,
-4773.913043478504,
4997653.043478261
],
"zarr_conventions": [
{
"description": "Spatial coordinate information",
"name": "spatial",
"schema_url": "https://raw.githubusercontent.com/zarr-conventions/spatial/refs/tags/v0.1/schema.json",
"spec_url": "https://github.com/zarr-conventions/spatial/blob/v0.1/README.md",
"uuid": "689b58e2-cf7b-45e0-9fff-9cfc0883d6b4"
},
{
"description": "Coordinate reference system information for geospatial data",
"name": "proj",
"schema_url": "https://raw.githubusercontent.com/zarr-conventions/proj/refs/tags/v0.1/schema.json",
"spec_url": "https://github.com/zarr-conventions/proj/blob/v0.1/README.md",
"uuid": "f17cb550-5864-4468-aeb7-f3180cfb622f"
}
]
},
"members": {
"angle": {
"attributes": {},
Expand Down Expand Up @@ -271,9 +313,65 @@
"storage_transformers": [],
"zarr_format": 3
},
"spatial_ref": {
"attributes": {
"crs_wkt": "PROJCS[\"WGS 84 / UTM zone 32N\",GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AUTHORITY[\"EPSG\",\"4326\"]],PROJECTION[\"Transverse_Mercator\"],PARAMETER[\"latitude_of_origin\",0],PARAMETER[\"central_meridian\",9],PARAMETER[\"scale_factor\",0.9996],PARAMETER[\"false_easting\",500000],PARAMETER[\"false_northing\",0],UNIT[\"metre\",1],AXIS[\"Easting\",EAST],AXIS[\"Northing\",NORTH],AUTHORITY[\"EPSG\",\"32632\"]]",
"false_easting": 500000.0,
"false_northing": 0.0,
"geographic_crs_name": "WGS 84",
"grid_mapping_name": "transverse_mercator",
"horizontal_datum_name": "World Geodetic System 1984",
"inverse_flattening": 298.257223563,
"latitude_of_projection_origin": 0.0,
"longitude_of_central_meridian": 9.0,
"longitude_of_prime_meridian": 0.0,
"prime_meridian_name": "Greenwich",
"projected_crs_name": "WGS 84 / UTM zone 32N",
"reference_ellipsoid_name": "WGS 84",
"scale_factor_at_central_meridian": 0.9996,
"semi_major_axis": 6378137.0,
"semi_minor_axis": 6356752.314245179,
"spatial_ref": "PROJCS[\"WGS 84 / UTM zone 32N\",GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AUTHORITY[\"EPSG\",\"4326\"]],PROJECTION[\"Transverse_Mercator\"],PARAMETER[\"latitude_of_origin\",0],PARAMETER[\"central_meridian\",9],PARAMETER[\"scale_factor\",0.9996],PARAMETER[\"false_easting\",500000],PARAMETER[\"false_northing\",0],UNIT[\"metre\",1],AXIS[\"Easting\",EAST],AXIS[\"Northing\",NORTH],AUTHORITY[\"EPSG\",\"32632\"]]"
},
"chunk_grid": {
"configuration": {
"chunk_shape": []
},
"name": "regular"
},
"chunk_key_encoding": {
"configuration": {
"separator": "/"
},
"name": "default"
},
"codecs": [
{
"configuration": {
"endian": "little"
},
"name": "bytes"
},
{
"configuration": {
"checksum": false,
"level": 0
},
"name": "zstd"
}
],
"data_type": "int64",
"dimension_names": null,
"fill_value": 0,
"node_type": "array",
"shape": [],
"storage_transformers": [],
"zarr_format": 3
},
"sun_angles": {
"attributes": {
"_FillValue": "AAAAAAAA+H8="
"_FillValue": "AAAAAAAA+H8=",
"grid_mapping": "spatial_ref"
},
"chunk_grid": {
"configuration": {
Expand Down Expand Up @@ -351,7 +449,8 @@
},
"viewing_incidence_angles": {
"attributes": {
"_FillValue": "AAAAAAAA+H8="
"_FillValue": "AAAAAAAA+H8=",
"grid_mapping": "spatial_ref"
},
"chunk_grid": {
"configuration": {
Expand Down
Loading
Loading