diff --git a/apps/docs/content/guides/database/replication/bigquery.mdx b/apps/docs/content/guides/database/replication/bigquery.mdx index c583f3a095ec9..b224b2185cbcc 100644 --- a/apps/docs/content/guides/database/replication/bigquery.mdx +++ b/apps/docs/content/guides/database/replication/bigquery.mdx @@ -29,17 +29,22 @@ Before configuring BigQuery as a destination, set up the following in Google Clo 3. **GCP service account key**: Create a [service account](https://cloud.google.com/iam/docs/keys-create-delete) with appropriate permissions - Go to **IAM & Admin > Service Accounts** - Click **Create Service Account** - - Grant the "BigQuery Data Editor" role + - Grant the "BigQuery Data Editor" and "BigQuery Job User" roles - Create and download the JSON key file Required permissions: - `bigquery.datasets.get` +- `bigquery.jobs.create` - `bigquery.tables.create` +- `bigquery.tables.delete` - `bigquery.tables.get` - `bigquery.tables.getData` +- `bigquery.tables.list` - `bigquery.tables.update` - `bigquery.tables.updateData` +- `bigquery.routines.get` +- `bigquery.routines.list` ## Configure BigQuery as a destination @@ -89,15 +94,21 @@ BigQuery replication requires each source table to have a primary key, and the p BigQuery primary keys are `NOT ENFORCED`, and BigQuery CDC supports composite primary keys with up to 16 columns. Your source primary key must stay unique and non-null because BigQuery uses it to match CDC rows. -Source tables must also use a BigQuery-compatible Postgres `REPLICA IDENTITY` setting: +Source tables must also use a BigQuery-compatible Postgres `REPLICA IDENTITY` setting. Most tables can keep the Postgres default, as long as they have a primary key and all primary-key columns are included in the publication. -- `DEFAULT` with a primary key is supported and works for most tables. -- `FULL` is supported and is recommended for tables with large `text`, `jsonb`, `bytea`, or other values that Postgres may store out-of-line using TOAST. -- `USING INDEX`, `NOTHING`, or `DEFAULT` without a primary key are not supported for BigQuery replication. +| Source table setting | BigQuery support | Guidance | +| ------------------------------------------------ | ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `REPLICA IDENTITY DEFAULT` with a primary key | Supported | Recommended for most tables. BigQuery uses the replicated source primary key to apply upserts and deletes. | +| `REPLICA IDENTITY FULL` | Supported | Recommended for tables with large `text`, `jsonb`, `bytea`, or other values that Postgres may store out-of-line using TOAST, especially when those rows update. | +| `REPLICA IDENTITY USING INDEX` | Not supported | BigQuery CDC rows are keyed by the source primary key, not by an alternative unique index. | +| `REPLICA IDENTITY NOTHING` | Not supported | Updates and deletes do not include enough row identity for BigQuery to apply them safely. | +| `REPLICA IDENTITY DEFAULT` without a primary key | Not supported | BigQuery requires a source primary key. | For a general explanation of how replica identity affects update and delete events, see [How does replica identity affect updates and deletes?](/docs/guides/database/replication/external-replication-faq#how-does-replica-identity-affect-updates-and-deletes). -For updates, Postgres does not always send a complete old row through logical replication. It can also mark unchanged toasted values as `unchanged toast` instead of resending the value. The replication pipeline can reconstruct a complete update when the old row image contains the missing value, which is reliable with `REPLICA IDENTITY FULL`. BigQuery CDC upserts require a complete new row, so updates can fail for tables with toasted columns if the pipeline receives only a partial update row. +For updates, Postgres does not always send a complete old row through logical replication. It can also mark unchanged toasted values as `unchanged toast` instead of resending the value. BigQuery CDC upserts require a complete new row because omitted columns are not preserved in the destination. The replication pipeline can reconstruct a complete update when the old row image contains the missing value, which is reliable with `REPLICA IDENTITY FULL`. + +If a BigQuery pipeline fails with an error about a partial update row, set `REPLICA IDENTITY FULL` on the affected source table and restart the pipeline. Changing replica identity only affects new WAL records, so a retained update that was written before the change may still need to be skipped by recreating the pipeline or re-copying the affected table. Check a table's current replica identity: @@ -138,16 +149,22 @@ Schema change support for BigQuery is currently in beta. External replication su Supported schema changes: -- Adding a column +- Adding a nullable column - Removing a column - Renaming a column +- Dropping a `NOT NULL` constraint +- Setting or dropping supported column default metadata -Unsupported schema changes: +Unsupported or limited schema changes: - Changing a column's data type -- Replicating column default values +- Adding `NOT NULL` with `SET NOT NULL` +- Filling existing rows for `ADD COLUMN ... DEFAULT` +- Unsupported default expressions + +BigQuery requires added columns to be nullable. When a replicated `ADD COLUMN` includes a default, external replication can apply supported default metadata for future rows, but BigQuery does not backfill existing rows through that DDL. Existing destination rows remain `NULL` unless you run a separate backfill. -We plan to expand schema change support over time as the feature evolves. +Supported defaults are best-effort translations to BigQuery SQL. Unsupported defaults are skipped with a warning instead of failing replication. ## Limitations diff --git a/apps/docs/content/guides/database/replication/external-replication-faq.mdx b/apps/docs/content/guides/database/replication/external-replication-faq.mdx index cc09c6336bf59..b511cb07198f2 100644 --- a/apps/docs/content/guides/database/replication/external-replication-faq.mdx +++ b/apps/docs/content/guides/database/replication/external-replication-faq.mdx @@ -45,11 +45,13 @@ Schema change support is currently in beta and limited to the BigQuery destinati Supported schema changes: -- Adding a column +- Adding a nullable column - Removing a column - Renaming a column +- Dropping a `NOT NULL` constraint +- Setting or dropping supported column default metadata -External replication does not currently support changing column data types or replicating column default values. See [BigQuery schema change support](/docs/guides/database/replication/bigquery#schema-change-support) for details. +External replication does not currently support changing column data types, adding `NOT NULL` with `SET NOT NULL`, or filling existing destination rows for `ADD COLUMN ... DEFAULT`. Supported defaults are applied as destination metadata for future rows where BigQuery can represent them. See [BigQuery schema change support](/docs/guides/database/replication/bigquery#schema-change-support) for details. ## What happens when you disable external replication? @@ -69,6 +71,14 @@ Common reasons: Check your publication settings and verify your table meets the requirements. +## Why are partitioned tables replicated as separate tables? + +Postgres controls this with the publication's `publish_via_partition_root` setting. If the setting is `false`, or if you created the publication manually with SQL and did not set it, Postgres publishes changes from the leaf partitions. External replication then creates destination tables for those leaf partitions. If `publish_via_partition_root = true`, Postgres publishes changes as the partition root, so external replication treats the partition hierarchy as the published partition root. + +Publications created from the Dashboard replication flow use `publish_via_partition_root = true`. + +See [Partitioned tables](/docs/guides/database/replication/external-replication-setup#partitioned-tables) for examples and the full behavior. + ## How does replica identity affect updates and deletes? If inserts replicate but updates or deletes fail, check the table's `REPLICA IDENTITY` setting. @@ -132,6 +142,20 @@ Pipeline failures occur during the streaming phase when an error happens while r See [Handling errors](/docs/guides/database/replication/external-replication-monitoring#handling-errors) for more details. +## Why is replication lag increasing? + +Lag increases when Postgres produces WAL faster than the pipeline can confirm it has processed. Common causes include a slow or rate-limited destination, a pipeline issue, heavy source database activity, long transactions, network latency between the pipeline and source database, or a stopped/disconnected pipeline. + +Open [**Database > Replication**](/dashboard/project/_/database/replication), click **View status**, and check **Waiting to sync**, **Room before pausing**, **Last check-in**, **Connected**, and **Slot status**. See [Dealing with replication lag](/docs/guides/database/replication/external-replication-monitoring#dealing-with-replication-lag) for the full investigation and response flow. + +## What does a `Lost` slot status mean? + +`Lost` means Postgres has already removed WAL files that the pipeline's replication slot needed. The pipeline cannot continue from that slot. + +You can recreate the pipeline, or open the pipeline's **Advanced settings**, set **Invalidated slot behavior** to **Recreate**, and restart the pipeline. On restart, the pipeline creates a new replication slot and starts replication from scratch for all tables. This is required for consistency because the old slot can no longer provide every change the pipeline missed. + +See [Slot statuses](/docs/guides/database/replication/external-replication-monitoring#slot-statuses) for all slot states and what to do next. + ## Why is a table in error state? Table errors occur during the copy phase. To recover, click **View status**, find the affected table, and reset the table state. This will restart the table copy from the beginning. diff --git a/apps/docs/content/guides/database/replication/external-replication-monitoring.mdx b/apps/docs/content/guides/database/replication/external-replication-monitoring.mdx index 12e460f88cff8..eb22ca4c65c85 100644 --- a/apps/docs/content/guides/database/replication/external-replication-monitoring.mdx +++ b/apps/docs/content/guides/database/replication/external-replication-monitoring.mdx @@ -55,7 +55,35 @@ height={2146} #### Replication lag metrics -The status page shows replication lag metrics that help you determine how fast your pipeline is replicating data. These metrics are loaded directly from Postgres itself. +The status page shows replication lag metrics that help you determine how far the pipeline is behind Postgres. These metrics are loaded directly from Postgres replication slot state. + +The destinations list also shows a compact lag value. This value is byte-based: it shows how much WAL the pipeline has not confirmed as flushed yet. A value of **Caught up** means the pipeline has confirmed every change currently available for its slot. + +The detailed status page shows: + +| Metric | What it means | What to watch for | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Waiting to sync** | Bytes of WAL between the pipeline's confirmed flush position and the current Postgres WAL position. This is the main byte-based replication lag. | A value that keeps growing means the pipeline is receiving changes more slowly than Postgres produces them. | +| **Room before pausing** | How much WAL can still accumulate before Postgres can no longer safely keep all WAL needed by the replication slot. This is controlled by `max_slot_wal_keep_size`. | A small or shrinking value means the slot is getting closer to being invalidated. `Unlimited` means Postgres is not reporting a slot WAL retention limit. | +| **Last check-in** | How long it has been since the pipeline last sent replication feedback to Postgres. | An old value can mean the pipeline is stopped, disconnected, overloaded, or unable to make progress. | +| **Connected** | Whether the pipeline's replication slot is active and currently being used. | `Not connected` while the pipeline should be running usually means you should check pipeline status and logs. | +| **Slot status** | How safely Postgres is keeping the WAL files the pipeline still needs. | `Unreserved` and `Lost` require action. See [Slot statuses](#slot-statuses). | + +External replication uses one main pipeline replication slot for ongoing changes. During the initial copy phase, it can also create temporary table-sync replication slots. These temporary slots let multiple tables copy in parallel, make large table copies faster, and allow individual tables to be retried or copied again without restarting the whole pipeline. + +Temporary table-sync slots show the same kind of lag and slot health metrics while they are active. After a table finishes copying and catches up, its temporary slot is removed and ongoing changes continue through the main pipeline slot. For overall replication health, focus first on the main pipeline slot. + +#### Slot statuses + +Replication slot status tells you whether Postgres is still retaining the WAL that the pipeline needs to continue from its current position. + +| Status | Meaning | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Reserved** | Healthy. Postgres is keeping the WAL files this pipeline's replication slot needs, and they are within the normal WAL size limit. | +| **Extended** | Healthy, but growing. The slot is holding on to more WAL than usual, but Postgres is still keeping everything the pipeline needs. | +| **Unreserved** | At risk. Postgres is no longer reserving all WAL files this pipeline's replication slot needs. If the pipeline does not catch up soon, those files may be removed. | +| **Lost** | Broken. Some WAL files this pipeline's replication slot needs have already been removed. The pipeline can no longer continue from this slot. Recreate the pipeline, or set **Invalidated slot behavior** to **Recreate** in the pipeline's advanced settings and restart it. | +| **Unknown** | Postgres reported an unknown or unavailable state for this pipeline's replication slot. | #### Table states @@ -69,6 +97,57 @@ The pipeline status page also shows the state of individual tables being replica | **Live** | Table is now replicating data in near real-time | | **Error** | Table has experienced an error during replication | +### Dealing with replication lag + +Replication lag means the pipeline is behind the source database. Some lag is expected during the initial copy phase, after a burst of writes, or after restarting a stopped pipeline. Lag becomes a problem when it keeps increasing, when **Room before pausing** is running low, or when the slot status moves to **Unreserved** or **Lost**. + +Lag can come from several places: + +- **Destination throughput**: The destination is slow, rate-limited, unavailable, or rejecting writes. +- **Pipeline throughput**: The pipeline is overloaded, processing a very large transaction, or not performing as expected for the project workload. +- **Source database activity**: Postgres is producing WAL faster than the pipeline can consume it, often during bulk writes, migrations, or backfill jobs. +- **Network latency**: Latency or instability between the pipeline and source database can slow down WAL streaming. +- **Stopped or disconnected pipeline**: When a pipeline is stopped, disconnected, or failed, Postgres keeps WAL for the slot until the retention limit is reached. +- **Slow initial table copy**: A temporary table-sync slot can fall behind if a table is copied more slowly than new changes are written to that table. + +#### Initial copy and table-sync slots + +A common initial sync issue happens when a large or busy table is still in **Copying** while new rows keep being inserted or updated. The temporary table-sync slot needs to keep the changes that happen during the copy. If the copy is too slow compared to the table's write rate, the slot can move to **Unreserved** and then **Lost** if Postgres removes changes the copy still needs. + +When a table-sync slot is lost, the affected table needs to be copied again. Tune the copy settings, then retry the table copy: + +- Increase **Copy connections per table** when one large table is the bottleneck. This lets the pipeline copy chunks of that table in parallel, up to the point where the source database or network becomes the limit. +- Increase **Table sync workers** when several tables need to copy at the same time. Each worker can copy one table, and each worker uses an additional temporary replication slot during initial sync. +- If possible, run the initial copy during a quieter write period or reduce bulk writes until the table reaches **Live**. + +After the affected table finishes copying and catches up, the temporary slot is deleted. The table then continues through the main pipeline replication slot. + +#### Investigate the lag + +1. Open [**Database > Replication**](/dashboard/project/_/database/replication) and check the destination's lag column. +2. Click **View status** and check **Waiting to sync**, **Room before pausing**, **Last check-in**, **Connected**, and **Slot status**. +3. Check table states. Tables in **Copying** can create temporary lag while the initial snapshot catches up to live changes. If a table-sync slot is **Unreserved** or **Lost**, tune copy parallelism and retry the affected table copy. +4. Open [**Logs > Replication**](/dashboard/project/_/logs/explorer) and look for destination errors, retries, rate limits, schema errors, or repeated restarts. +5. Compare the lag trend with recent database activity, such as imports, migrations, bulk updates, or long transactions. + +#### Respond based on the slot status + +| Slot status | What to do | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Reserved** | If **Waiting to sync** is stable or decreasing, continue monitoring. If it keeps increasing, check destination write performance, logs, and whether the publication includes more tables or write volume than expected. | +| **Extended** | Treat it as an early warning. Confirm the pipeline is connected, check logs for retries or destination slowness, and reduce avoidable write bursts if possible until the pipeline catches up. | +| **Unreserved** | Act quickly. The slot is at risk of losing required WAL. Check whether the pipeline is connected and making progress, fix destination or pipeline errors, and contact support if the lag continues to grow. | +| **Lost** | The pipeline cannot continue from the existing slot because required WAL has been removed. Recreate the pipeline, or set **Invalidated slot behavior** to **Recreate** in the pipeline's advanced settings and restart the pipeline. This creates a new slot and starts replication from scratch for all tables. | +| **Unknown** | Check replication logs for errors or missing slot details. If the status remains unknown while the pipeline should be running, contact support with the pipeline ID and recent log details. | + +#### Reduce future lag risk + +- Keep publications focused on the tables and operations you need at the destination. +- Avoid leaving pipelines stopped for long periods while the source database is still receiving writes. +- Schedule bulk updates, imports, and migrations during lower-traffic windows when possible. +- For BigQuery, verify that service account permissions, table requirements, and replica identity settings match the [BigQuery destination guide](/docs/guides/database/replication/bigquery). +- If the initial copy is the bottleneck, review **Table sync workers** and **Copy connections per table** in the pipeline's advanced settings. Increasing them can speed up copying, but it uses more database connections and replication slots. + ### Handling errors Errors can occur at two levels: per table or per pipeline. diff --git a/apps/docs/content/guides/database/replication/external-replication-setup.mdx b/apps/docs/content/guides/database/replication/external-replication-setup.mdx index f1c58b4303d85..f82edd4f4579f 100644 --- a/apps/docs/content/guides/database/replication/external-replication-setup.mdx +++ b/apps/docs/content/guides/database/replication/external-replication-setup.mdx @@ -88,6 +88,44 @@ create publication pub_recent_orders for table orders where (created_at > '2024-01-01'); ``` +##### Partitioned tables + +External replication follows Postgres publication semantics for partitioned tables. The `publish_via_partition_root` publication setting controls whether changes from partitions are emitted as the partition root or as the leaf partitions. + +| Publication setting | What gets replicated | Destination shape | +| ------------------------------------------ | ---------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------- | +| `publish_via_partition_root = true` | Rows from the published partition root, including rows stored in its leaf partitions | One table matching the published partition root | +| `publish_via_partition_root = false` | Rows from the leaf partitions under the published partition root | One table per replicated leaf partition | +| Not set in SQL | Same as `false`, because Postgres defaults `publish_via_partition_root` to `false` | One table per replicated leaf partition | +| Publishing an individual leaf partition | The leaf partition itself, regardless of `publish_via_partition_root` | One table for that leaf partition | +| `FOR ALL TABLES` or `FOR TABLES IN SCHEMA` | Partition roots plus regular tables when `true`; leaf partitions plus regular tables when `false` or unset | Destination tables follow the effective Postgres publication table list | + +For example, if `orders` is partitioned by month: + +```sql +-- Replicate the whole partition hierarchy as the parent table. +create publication pub_orders_root +for table orders +with (publish_via_partition_root = true); + +-- Replicate each leaf partition as its own table. +create publication pub_orders_leaves +for table orders +with (publish_via_partition_root = false); +``` + +Use `publish_via_partition_root = true` when you want analytics queries to read from a single destination table that has the parent table's schema. Use `false` when each partition should remain a separate destination table. + +Publications created from the Dashboard replication flow use `publish_via_partition_root = true`. If you create or alter a publication manually with SQL, set this option explicitly so the destination shape matches what you expect. + +On Postgres 15 and newer, row filters on partition publications apply during both the initial copy and streaming phases. External replication uses the row filter attached to the effective publication table entry: the published partition root when `publish_via_partition_root = true`, and the published leaf relation when `publish_via_partition_root = false`. + + + +With `publish_via_partition_root = true`, truncating an individual leaf partition is not replicated as a truncate event for the published parent. If you need truncate replication, run `TRUNCATE` on the published partition root. + + + #### Viewing publications in the Dashboard After creating a publication via SQL, you can view it in the Dashboard: @@ -132,15 +170,17 @@ Follow these steps to configure your destination. Each destination has its own s 5. Optionally expand **Advanced settings** to tune pipeline behavior. These settings apply to the pipeline rather than the destination: - | Setting | Default | Description | - | ------------------------------ | -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | - | **Batch wait time** | `10000` milliseconds | Maximum time the pipeline waits to collect additional changes before flushing them. Lower values reduce replication latency. Higher values can improve batching efficiency. | - | **Table sync workers** | `4` workers | Number of tables copied in parallel during the initial snapshot phase. Each worker uses one replication slot, up to `N + 1` total replication slots while syncing. | - | **Copy connections per table** | `2` connections | Number of parallel database connections each table copy can use during the initial sync. Increasing this can speed up large table copies, but uses more database connections. | - | **Invalidated slot behavior** | `Error` | What the pipeline does when its replication slot is invalidated. **Error** blocks startup so you can recover manually. **Recreate** rebuilds the slot and starts replication from scratch. | + | Setting | Default | Description | + | ------------------------------ | -------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | + | **Batch wait time** | `10000` milliseconds | Maximum time the pipeline waits to collect additional changes before flushing them. Lower values reduce replication latency. Higher values can improve batching efficiency. | + | **Table sync workers** | `4` workers | Number of tables copied in parallel during the initial snapshot phase. Each worker uses one replication slot, up to `N + 1` total replication slots while syncing. | + | **Copy connections per table** | `2` connections | Number of parallel database connections each table copy can use during the initial sync. Increasing this can speed up large table copies, but uses more database connections. | + | **Invalidated slot behavior** | `Error` | What the pipeline does when its replication slot is invalidated. **Error** blocks startup so you can recover manually. **Recreate** rebuilds the slot on the next pipeline restart and starts replication from scratch for all tables. | Leave these settings at their defaults unless you need to tune initial copy speed, latency, or recovery behavior. + Use **Invalidated slot behavior** carefully. If **Recreate** is selected and the pipeline restarts after Postgres has invalidated the main replication slot, the pipeline creates a new slot and copies all replicated tables again. This full restart is required for consistency because the old slot can no longer provide every change the pipeline missed. + 6. Click **Create and start** to begin replication Your replication pipeline now starts copying data from your database to your destination. diff --git a/apps/docs/content/guides/getting-started.mdx b/apps/docs/content/guides/getting-started.mdx index 7559d93015f22..45a0a7892cc69 100644 --- a/apps/docs/content/guides/getting-started.mdx +++ b/apps/docs/content/guides/getting-started.mdx @@ -10,7 +10,7 @@ hideToc: true
- + - + Use the Supabase CLI to develop locally and collaborate between teams. diff --git a/apps/docs/content/guides/realtime/limits.mdx b/apps/docs/content/guides/realtime/limits.mdx index edea5d6dc2d1b..5c45b111e4d7c 100644 --- a/apps/docs/content/guides/realtime/limits.mdx +++ b/apps/docs/content/guides/realtime/limits.mdx @@ -23,6 +23,7 @@ Upgrade your plan to increase your limits. Without a spend cap, or on an Enterpr | **Channels per connection** | 100 | 100 | 100 | 100 | 100+ | | **Presence keys per object** | 10 | 10 | 10 | 10 | 10+ | | **Presence messages per second** | 20 | 50 | 1,000 | 1,000 | 1,000+ | +| **Presence calls per client, per 30 seconds** | 5 | 5 | 5 | 5 | 5 | | **Broadcast payload size** | 256 KB | 3,000 KB | 3,000 KB | 3,000 KB | 3,000+ KB | | **Postgres change payload size ([**read more**](#postgres-changes-payload-limit))** | 1,024 KB | 1,024 KB | 1,024 KB | 1,024 KB | 1,024+ KB | diff --git a/apps/docs/content/troubleshooting/relation-objects-does-not-exist-error-during-storage-uploads-8f21f0.mdx b/apps/docs/content/troubleshooting/relation-objects-does-not-exist-error-during-storage-uploads-8f21f0.mdx new file mode 100644 index 0000000000000..69de7d3fbb2e5 --- /dev/null +++ b/apps/docs/content/troubleshooting/relation-objects-does-not-exist-error-during-storage-uploads-8f21f0.mdx @@ -0,0 +1,30 @@ +--- +title = "Postgres error 'relation objects does not exist' during Storage uploads" +date_created = "2026-06-10T15:05:10+00:00" +topics = [ "auth", "database", "storage" ] +keywords = [] +[[errors]] +code = "42P01" +message = "relation 'objects' does not exist" + +[[errors]] +http_status_code = 500 +message = "Internal Server Error" +--- + +If authenticated users receive a `500 Internal Server Error` during file uploads and your database logs report `error: relation "objects" does not exist`, it typically indicates the database cannot locate the required internal storage tables. + +**Why Does This Happen?** +This error occurs when the `authenticated` role's `search_path` does not include the `storage` schema. Because the schema is missing from the path, the database fails to find the `objects` table (which resides in `storage.objects`) during the upload process. + +**How to Fix:** +You can resolve this by updating the role configuration via the [SQL Editor](/dashboard/project/_/sql/new): + +1. Execute the following command to append the storage schema to the role's search path: + `ALTER ROLE authenticated SET search_path = public, storage;` +2. Verify that uploads from authenticated users now succeed. + +**Best Practices:** + +- To maintain compatibility with default schema permissions, avoid creating custom Postgres roles for application users. +- Instead, use custom claims within `auth.users.raw_app_meta_data` to manage user-specific logic or permissions. diff --git a/apps/studio/components/interfaces/Database/Replication/DeleteDestination.tsx b/apps/studio/components/interfaces/Database/Replication/DeleteDestination.tsx index 2489f40535598..0fddb5a6b1890 100644 --- a/apps/studio/components/interfaces/Database/Replication/DeleteDestination.tsx +++ b/apps/studio/components/interfaces/Database/Replication/DeleteDestination.tsx @@ -24,7 +24,7 @@ export const DeleteDestination = ({ confirmLabel={isLoading ? 'Deleting...' : `Delete destination`} confirmPlaceholder="Type in name of destination" confirmString={name ?? 'Unknown'} - text={`This will delete the destination "${name}"`} + text={`This will delete the destination "${name}".`} alert={{ title: 'You cannot recover this destination once deleted.' }} onCancel={() => setVisible(!visible)} onConfirm={onDelete} diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx index 3f909475df8bd..cba27702b7abc 100644 --- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx @@ -43,7 +43,7 @@ export const AdvancedSettings = ({
Advanced settings - Optional performance tuning + Optional settings to control the pipeline in more depth
@@ -165,7 +165,7 @@ export const AdvancedSettings = ({ - {destinationType !== 'Read Replica' && ( -

- External replication is in alpha. Expect rapid changes and possible breaking updates.{' '} + {selectedOption?.isAlpha && ( +

+ This destination type is in alpha and may change while we iterate.{' '} Leave feedback diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/ReadReplicaForm/ReadReplicaEligibilityWarnings.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/ReadReplicaForm/ReadReplicaEligibilityWarnings.tsx index ed01b16a504c1..370d235c8a84a 100644 --- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/ReadReplicaForm/ReadReplicaEligibilityWarnings.tsx +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/ReadReplicaForm/ReadReplicaEligibilityWarnings.tsx @@ -65,7 +65,7 @@ export const ReadReplicaEligibilityWarnings = () => { if (hasOverdueInvoices) { return ( -

Please resolve all outstanding invoices first before deploying a new read replica

+

Please resolve all outstanding invoices first before deploying a new read replica.

@@ -81,7 +81,7 @@ export const ReadReplicaEligibilityWarnings = () => { >

Projects provisioned by other cloud providers currently will not be able to use read - replicas + replicas.

{ ) } @@ -108,7 +108,7 @@ export const ReadReplicaEligibilityWarnings = () => { type="warning" title="Read replicas can only be deployed with projects on Postgres version 15 and above" > -

If you'd like to use read replicas, please contact us via support

+

If you'd like to use read replicas, please contact us via support.

diff --git a/apps/studio/components/interfaces/Database/Replication/ReadReplicas/ReadReplicaDetails.tsx b/apps/studio/components/interfaces/Database/Replication/ReadReplicas/ReadReplicaDetails.tsx index f6b43c32a524b..c40af9169b70a 100644 --- a/apps/studio/components/interfaces/Database/Replication/ReadReplicas/ReadReplicaDetails.tsx +++ b/apps/studio/components/interfaces/Database/Replication/ReadReplicas/ReadReplicaDetails.tsx @@ -137,7 +137,7 @@ export const ReadReplicaDetails = () => { isReactForm={false} layout="horizontal" label="Load Balancer URL" - description="RESTful endpoint for querying and managing your databases through your load balancer" + description="RESTful endpoint for querying and managing your databases through your load balancer." > @@ -164,7 +164,7 @@ export const ReadReplicaDetails = () => { isReactForm={false} layout="horizontal" label="Compute Size" - description="Size of replica will be identical to the primary database" + description="Size of replica will be identical to the primary database." > diff --git a/apps/studio/components/interfaces/Database/Replication/ReplicationDiagram/Edges.tsx b/apps/studio/components/interfaces/Database/Replication/ReplicationDiagram/Edges.tsx index 299cced7593c1..84eebdfbc91eb 100644 --- a/apps/studio/components/interfaces/Database/Replication/ReplicationDiagram/Edges.tsx +++ b/apps/studio/components/interfaces/Database/Replication/ReplicationDiagram/Edges.tsx @@ -1,19 +1,85 @@ import { BaseEdge, EdgeLabelRenderer, getSmoothStepPath, type EdgeProps } from '@xyflow/react' import { useParams } from 'common' -import { Loader2, X } from 'lucide-react' -import { cn, Tooltip, TooltipContent, TooltipTrigger } from 'ui' +import { ArrowRight, Loader2, Square, X, type LucideIcon } from 'lucide-react' +import { useMemo } from 'react' +import { cn } from 'ui' -import { useReplicationLagQuery } from '@/data/read-replicas/replica-lag-query' -import { formatDatabaseID } from '@/data/read-replicas/replicas.utils' -import { useSelectedProjectQuery } from '@/hooks/misc/useSelectedProject' +import { getStatusName } from '../Pipeline.utils' +import { STATUS_REFRESH_FREQUENCY_MS } from '../Replication.constants' +import { REPLICA_STATUS } from '@/components/interfaces/Settings/Infrastructure/InfrastructureConfiguration/InstanceConfiguration.constants' +import { useReadReplicasQuery } from '@/data/read-replicas/replicas-query' +import { useReplicationPipelineStatusQuery } from '@/data/replication/pipeline-status-query' +import { useReplicationPipelinesQuery } from '@/data/replication/pipelines-query' +import { + PipelineStatusRequestStatus, + usePipelineRequestStatus, +} from '@/state/replication-pipeline-request-status' type EdgeData = { type: string identifier: string + shiftEdgeEnd: boolean +} + +interface ReplicationState { isComingUp: boolean isReplicating: boolean isFailed: boolean - shiftEdgeEnd: boolean +} + +interface EdgeVisual { + Icon: LucideIcon + // CSS color shared by the icon and the connecting line so they always match. + color: string + opacity: number + dashArray: string + shouldAnimate: boolean + shouldSpin?: boolean + isFilled?: boolean + strokeWidth?: number +} + +// Picks the icon + line appearance for a replication state. Both the icon and the line are derived +// here from the same state so they always stay in sync. We deliberately don't surface lag: the line +// just communicates whether data is moving, stopped, starting, or broken. +const getEdgeVisual = ({ isComingUp, isReplicating, isFailed }: ReplicationState): EdgeVisual => { + if (isFailed) { + return { + Icon: X, + color: 'hsl(var(--destructive-default))', + opacity: 1, + dashArray: '5 5', + shouldAnimate: false, + strokeWidth: 4, + } + } + if (isComingUp) { + return { + Icon: Loader2, + color: 'hsl(var(--foreground-light))', + opacity: 1, + dashArray: '5', + shouldAnimate: true, + shouldSpin: true, + } + } + if (isReplicating) { + return { + Icon: ArrowRight, + color: 'hsl(var(--brand-default))', + opacity: 1, + dashArray: '5', + shouldAnimate: true, + } + } + return { + Icon: Square, + color: 'hsl(var(--foreground-lighter))', + opacity: 0.5, + dashArray: '5 5', + shouldAnimate: false, + isFilled: true, + } } export const SmoothstepEdge = ({ @@ -27,12 +93,58 @@ export const SmoothstepEdge = ({ markerEnd, data, }: EdgeProps) => { - const { ref } = useParams() - const { data: project } = useSelectedProjectQuery() + const { ref: projectRef = 'default' } = useParams() + const { type, identifier, shiftEdgeEnd } = (data || {}) as EdgeData + const isReplica = type === 'replica' - const { type, identifier, isComingUp, isReplicating, isFailed, shiftEdgeEnd } = (data || - {}) as EdgeData - const formattedId = type === 'replica' ? formatDatabaseID(identifier ?? '') : identifier + // Subscribe to the same live status the nodes use, so the line and the node update together. + const { data: databases = [] } = useReadReplicasQuery( + { projectRef }, + { enabled: isReplica, refetchInterval: STATUS_REFRESH_FREQUENCY_MS } + ) + const replica = databases.find((x) => x.identifier === identifier) + + const { data: pipelinesData } = useReplicationPipelinesQuery( + { projectRef }, + { enabled: !isReplica } + ) + const pipeline = (pipelinesData?.pipelines ?? []).find( + (p) => p.destination_id.toString() === identifier + ) + const { data: pipelineStatusData } = useReplicationPipelineStatusQuery( + { projectRef, pipelineId: pipeline?.id }, + { enabled: !isReplica && !!pipeline?.id, refetchInterval: STATUS_REFRESH_FREQUENCY_MS } + ) + const { getRequestStatus } = usePipelineRequestStatus() + const requestStatus = pipeline?.id + ? getRequestStatus(pipeline.id) + : PipelineStatusRequestStatus.None + + const replicationState = useMemo(() => { + if (isReplica) { + const status = replica?.status + return { + isReplicating: status === 'ACTIVE_HEALTHY', + isComingUp: + status !== undefined && + [ + REPLICA_STATUS.COMING_UP, + REPLICA_STATUS.INIT_READ_REPLICA, + REPLICA_STATUS.UNKNOWN, + ].includes(status), + isFailed: + status !== undefined && + [REPLICA_STATUS.ACTIVE_UNHEALTHY, REPLICA_STATUS.INIT_FAILED].includes(status), + } + } + const isTransitioning = requestStatus !== PipelineStatusRequestStatus.None + const statusName = getStatusName(pipelineStatusData?.status) + return { + isReplicating: statusName === 'started' && !isTransitioning, + isComingUp: isTransitioning || statusName === 'starting' || statusName === 'stopping', + isFailed: statusName === 'failed', + } + }, [isReplica, replica?.status, pipelineStatusData?.status, requestStatus]) const [edgePath, labelX, labelY] = getSmoothStepPath({ sourceX, @@ -43,69 +155,39 @@ export const SmoothstepEdge = ({ targetPosition, }) - const { - data: lagDuration, - isPending: isLoading, - isError, - } = useReplicationLagQuery( - { - id: identifier, - projectRef: ref, - connectionString: project?.connectionString, - }, - { enabled: type === 'replica' && isReplicating, refetchInterval: 10000 } - ) - const lagValue = Number(lagDuration?.toFixed(2) ?? 0).toLocaleString() - const hasReplicationLagData = data !== undefined && !isError && isReplicating + const { Icon, color, opacity, dashArray, shouldAnimate, shouldSpin, isFilled, strokeWidth } = + getEdgeVisual(replicationState) return ( <> - - - {isFailed && ( - -
- -
- - )} + - {(isComingUp || hasReplicationLagData) && ( - - - -
- {isLoading || isComingUp ? ( - - ) : ( -

{lagValue}s

- )} -
- - {!isComingUp && ( - - {isLoading - ? `Checking replication lag for replica ID: ${formattedId}` - : `Replication lag (seconds) for replica ID: ${formattedId}`} - - )} - - - )} + +
+ +
+ ) } diff --git a/apps/studio/components/interfaces/Database/Replication/ReplicationDiagram/Nodes.tsx b/apps/studio/components/interfaces/Database/Replication/ReplicationDiagram/Nodes.tsx index 0de96624b30bd..95af86ba5bef9 100644 --- a/apps/studio/components/interfaces/Database/Replication/ReplicationDiagram/Nodes.tsx +++ b/apps/studio/components/interfaces/Database/Replication/ReplicationDiagram/Nodes.tsx @@ -105,25 +105,23 @@ export const ReplicationNode = ({ id }: { id: string }) => {

{type}

- - -
-
-
- - - {statusName} - - + {(statusName === 'started' || statusName === 'failed') && ( + + +
+
+
+ + + {statusName} + + + )}

{destination?.name}

ID: {destination?.id}

diff --git a/apps/studio/components/interfaces/Database/Replication/ReplicationDiagram/index.tsx b/apps/studio/components/interfaces/Database/Replication/ReplicationDiagram/index.tsx index d18ef483d366d..f90e8741bff14 100644 --- a/apps/studio/components/interfaces/Database/Replication/ReplicationDiagram/index.tsx +++ b/apps/studio/components/interfaces/Database/Replication/ReplicationDiagram/index.tsx @@ -1,23 +1,17 @@ -import { useQueryClient } from '@tanstack/react-query' import { Background, ColorMode, ReactFlow, ReactFlowProvider, useReactFlow } from '@xyflow/react' import { useParams } from 'common' import { useTheme } from 'next-themes' import { useEffect, useMemo } from 'react' -import { getStatusName } from '../Pipeline.utils' import { PrimaryDatabaseNode, ReadReplicaNode, ReplicationNode } from './Nodes' import { getDagreGraphLayout } from './ReplicationDiagram.utils' import { useReadReplicasQuery } from '@/data/read-replicas/replicas-query' import { useReplicationDestinationsQuery } from '@/data/replication/destinations-query' -import { replicationKeys } from '@/data/replication/keys' -import { ReplicationPipelineStatusResponse } from '@/data/replication/pipeline-status-query' -import { useReplicationPipelinesQuery } from '@/data/replication/pipelines-query' import { timeout } from '@/lib/helpers' import '@xyflow/react/dist/style.css' import { SmoothstepEdge } from './Edges' -import { REPLICA_STATUS } from '@/components/interfaces/Settings/Infrastructure/InfrastructureConfiguration/InstanceConfiguration.constants' export const ReplicationDiagram = () => { return ( @@ -38,7 +32,6 @@ const edgeTypes = { smoothstep: SmoothstepEdge } const ReplicationDiagramContent = () => { const reactFlow = useReactFlow() const { resolvedTheme } = useTheme() - const queryClient = useQueryClient() const { ref: projectRef = 'default' } = useParams() const { data: databases = [], isSuccess: isSuccessReplicas } = useReadReplicasQuery({ @@ -54,8 +47,6 @@ const ReplicationDiagramContent = () => { }) const destinations = useMemo(() => data?.destinations ?? [], [data]) - const { data: pipelinesData } = useReplicationPipelinesQuery({ projectRef }) - const nodes = useMemo(() => { return [ { id: projectRef, type: 'primary', data: {}, position: { x: 0, y: 5 } }, @@ -75,68 +66,28 @@ const ReplicationDiagramContent = () => { }, [destinations, projectRef, readReplicas]) const edges = useMemo(() => { + const shiftEdgeEnd = readReplicas.length + destinations.length > 1 + return [ - ...readReplicas.map((x) => { - const isReplicating = x.status === 'ACTIVE_HEALTHY' - - return { - id: `${projectRef}-${x.identifier}`, - source: projectRef, - target: x.identifier, - type: 'smoothstep', - className: 'cursor-default!', - animated: isReplicating, - style: { - opacity: isReplicating ? 1 : 0.4, - strokeDasharray: isReplicating ? undefined : '5 5', - }, - data: { - type: 'replica', - identifier: x.identifier, - shiftEdgeEnd: readReplicas.length + destinations.length > 1, - isReplicating, - isComingUp: [ - REPLICA_STATUS.COMING_UP, - REPLICA_STATUS.INIT_READ_REPLICA, - REPLICA_STATUS.UNKNOWN, - ].includes(x.status), - isFailed: [REPLICA_STATUS.ACTIVE_UNHEALTHY, REPLICA_STATUS.INIT_FAILED].includes( - x.status - ), - }, - } - }), - ...destinations.map((x) => { - const pipeline = (pipelinesData?.pipelines ?? []).find((p) => p.destination_id === x.id) - const pipelineStatus = queryClient.getQueryData( - replicationKeys.pipelinesStatus(projectRef, pipeline?.id) - ) as ReplicationPipelineStatusResponse - const statusName = getStatusName(pipelineStatus?.status) - const isReplicating = statusName === 'started' - - return { - id: `${projectRef}-${x.id}`, - source: projectRef, - target: x.id.toString(), - type: 'smoothstep', - className: 'cursor-default!', - animated: isReplicating, - style: { - opacity: isReplicating ? 1 : 0.4, - strokeDasharray: isReplicating ? undefined : '5 5', - }, - data: { - type: 'etl', - identifier: x.id, - shiftEdgeEnd: readReplicas.length + destinations.length > 1, - isReplicating, - isComingUp: ['starting'].includes(statusName ?? ''), - isFailed: ['failed'].includes(statusName ?? ''), - }, - } - }), + ...readReplicas.map((x) => ({ + id: `${projectRef}-${x.identifier}`, + source: projectRef, + target: x.identifier, + type: 'smoothstep', + className: 'cursor-default!', + // The edge subscribes to live status itself (see Edges.tsx) so it stays in sync with nodes. + data: { type: 'replica', identifier: x.identifier, shiftEdgeEnd }, + })), + ...destinations.map((x) => ({ + id: `${projectRef}-${x.id}`, + source: projectRef, + target: x.id.toString(), + type: 'smoothstep', + className: 'cursor-default!', + data: { type: 'etl', identifier: x.id.toString(), shiftEdgeEnd }, + })), ] - }, [destinations, pipelinesData?.pipelines, projectRef, queryClient, readReplicas]) + }, [destinations, projectRef, readReplicas]) const backgroundPatternColor = resolvedTheme === 'dark' ? 'rgba(255, 255, 255, 0.3)' : 'rgba(0, 0, 0, 0.4)' diff --git a/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.tsx b/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.tsx index 915389d6033b7..d872bb4816fb0 100644 --- a/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.tsx +++ b/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.tsx @@ -47,6 +47,7 @@ import { UpdateVersionModal } from '../UpdateVersionModal' import { SlotLagMetrics } from './ReplicationPipelineStatus.types' import { getDisabledStateConfig } from './ReplicationPipelineStatus.utils' import { SlotLagMetricsInline, SlotLagMetricsList } from './SlotLagMetrics' +import { SlotConnectionIndicator, SlotStatusBadge, SlotStatusLegend } from './SlotStatus' import { TableReplicationRow } from './TableReplicationRow' import { AlertError } from '@/components/ui/AlertError' import { DropdownMenuItemTooltip } from '@/components/ui/DropdownMenuItemTooltip' @@ -344,16 +345,19 @@ export const ReplicationPipelineStatus = () => { {applyLagMetrics && (
-
+
-

Replication lag

+

Pipeline metrics

- Snapshot of how far this pipeline is trailing behind right now. + Live metrics on how this pipeline is doing right now.

-

- Updates every {refreshIntervalLabel} -

+
+ + + + +
{isStatusError && ( diff --git a/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.types.ts b/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.types.ts index 02aa0c694a9d3..568cadb6125fd 100644 --- a/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.types.ts +++ b/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.types.ts @@ -3,15 +3,29 @@ export type RetryPolicy = | { policy: 'manual_retry' } | { policy: 'timed_retry'; next_retry: string } +// WAL availability status reported by Postgres for the replication slot. Unrecognized future +// values from Postgres are surfaced as 'unknown'; the field is absent when not reported. +export type SlotWalStatus = 'reserved' | 'extended' | 'unreserved' | 'lost' | 'unknown' + export type SlotLagMetrics = { + active: boolean + wal_status?: SlotWalStatus restart_lsn_bytes: number confirmed_flush_lsn_bytes: number - safe_wal_size_bytes: number + // null means Postgres reports unlimited slot WAL retention. + safe_wal_size_bytes: number | null write_lag?: number flush_lag?: number + // Milliseconds since the destination last sent feedback. Present even when write/flush lag are not. + reply_time_lag?: number } -export type SlotLagMetricKey = keyof SlotLagMetrics +// Numeric metrics rendered as value tiles (kept separate from the non-numeric slot fields above). +export type SlotLagMetricKey = + | 'confirmed_flush_lsn_bytes' + | 'safe_wal_size_bytes' + | 'flush_lag' + | 'reply_time_lag' export type TableState = { table_id: number diff --git a/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.utils.tsx b/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.utils.tsx index 525b86de9efd8..68ebf1d39e24a 100644 --- a/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.utils.tsx +++ b/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.utils.tsx @@ -2,13 +2,11 @@ import dayjs from 'dayjs' import { Badge } from 'ui' import { getPipelineDisplayState, normalizePipelineStatusName } from '../Pipeline.utils' -import { RetryPolicy, TableState } from './ReplicationPipelineStatus.types' +import { RetryPolicy, SlotWalStatus, TableState } from './ReplicationPipelineStatus.types' import { ReplicationPipelineStatusData } from '@/data/replication/pipeline-status-query' import { formatBytes } from '@/lib/helpers' import { PipelineStatusRequestStatus } from '@/state/replication-pipeline-request-status' -const numberFormatter = new Intl.NumberFormat() - export const getStatusConfig = (state: TableState['state']) => { switch (state.name) { case 'queued': @@ -86,19 +84,19 @@ export const isValidRetryPolicy = (policy: any): policy is RetryPolicy => { const formatLagBytesValue = (value?: number) => { if (typeof value !== 'number' || Number.isNaN(value)) { - return { display: '—', detail: undefined } + return { display: 'n/a', detail: undefined } } + // Scale to the most readable unit (e.g. "4 GB"). We intentionally don't surface the raw byte + // count as a detail line, since it's unreadable at GB scale (e.g. "4,294,967,296 bytes"). const decimals = value < 1024 ? 0 : value < 1024 * 1024 ? 1 : 2 - const display = formatBytes(value, decimals) - const detail = `${numberFormatter.format(value)} bytes` - - return { display, detail } + return { display: formatBytes(value, decimals), detail: undefined } } +// Scale to a single readable unit (ms, s, min, h) based on size, with no precise sub-line. const formatLagDurationValue = (value?: number) => { if (typeof value !== 'number' || Number.isNaN(value)) { - return { display: '—', detail: undefined } + return { display: 'n/a', detail: undefined } } const sign = value < 0 ? '-' : '' @@ -111,29 +109,148 @@ const formatLagDurationValue = (value?: number) => { const seconds = duration.asSeconds() if (seconds < 60) { - const decimals = seconds >= 10 ? 1 : 2 - return { - display: `${sign}${seconds.toFixed(decimals)} s`, - detail: `${numberFormatter.format(value)} ms`, - } + return { display: `${sign}${seconds.toFixed(seconds >= 10 ? 1 : 2)} s`, detail: undefined } } const minutes = duration.asMinutes() if (minutes < 60) { - const roundedSeconds = Math.round(seconds) - return { - display: `${sign}${minutes.toFixed(minutes >= 10 ? 1 : 2)} min`, - detail: `${numberFormatter.format(roundedSeconds)} s`, - } + return { display: `${sign}${minutes.toFixed(minutes >= 10 ? 1 : 2)} min`, detail: undefined } } const hours = duration.asHours() - const roundedMinutes = Math.round(minutes) - return { - display: `${sign}${hours.toFixed(hours >= 10 ? 1 : 2)} h`, - detail: `${numberFormatter.format(roundedMinutes)} min`, - } + return { display: `${sign}${hours.toFixed(hours >= 10 ? 1 : 2)} h`, detail: undefined } } export const getFormattedLagValue = (type: 'bytes' | 'duration', value?: number) => type === 'bytes' ? formatLagBytesValue(value) : formatLagDurationValue(value) + +export type LagSeverity = 'normal' | 'warning' | 'critical' + +type SlotStatusBadgeVariant = 'success' | 'warning' | 'destructive' | 'default' + +interface WalStatusMeta { + label: string + variant: SlotStatusBadgeVariant + severity: LagSeverity + // Shown in the pipeline-level metrics panel. + description: string + // Shown in the per-table inline sync view where the slot belongs to a single table. + tableDescription: string +} + +// Plain-language meaning, color, and severity for each WAL status Postgres can report for a slot. +// `variant` drives the badge color; `severity` drives whether the list view raises a warning icon +// (e.g. "extended" is shown amber as a heads-up but isn't alarming on its own). +export const WAL_STATUS_META: Record = { + reserved: { + label: 'Reserved', + variant: 'success', + severity: 'normal', + description: + "Healthy. Your database is keeping the WAL files this pipeline's replication slot needs, and they are within the normal WAL size limit.", + tableDescription: + "Healthy. Your database is keeping the WAL files this table's replication slot needs, and they are within the normal WAL size limit.", + }, + extended: { + label: 'Extended', + variant: 'warning', + severity: 'normal', + description: + "Healthy, but growing. This pipeline's replication slot is holding on to more WAL than usual, but your database is still keeping everything it needs.", + tableDescription: + "Healthy, but growing. This table's replication slot is holding on to more WAL than usual, but your database is still keeping everything it needs.", + }, + unreserved: { + label: 'Unreserved', + variant: 'warning', + severity: 'warning', + description: + "At risk. Your database is no longer reserving all WAL files this pipeline's replication slot needs. If the pipeline does not catch up soon, those files may be removed.", + tableDescription: + "At risk. Your database is no longer reserving all WAL files this table's replication slot needs. If the pipeline does not catch up soon, those files may be removed.", + }, + lost: { + label: 'Lost', + variant: 'destructive', + severity: 'critical', + description: + "Broken. Some WAL files this pipeline's replication slot needs have already been removed. The pipeline can no longer continue from this slot. You can recreate a new pipeline, or set the invalidation behavior to recreate and restart the pipeline.", + tableDescription: + "Broken. Some WAL files this table's replication slot needs have already been removed. The pipeline can no longer continue from this slot. You can recreate a new pipeline, or set the invalidation behavior to recreate and restart the pipeline.", + }, + unknown: { + label: 'Unknown', + variant: 'default', + severity: 'normal', + description: + "Unknown. Your database reported an unknown state for this pipeline's replication slot.", + tableDescription: + "Unknown. Your database reported an unknown state for this table's replication slot.", + }, +} + +// Postgres reports no WAL status (restart_lsn is null) as "unknown" too, so fall back to it. +export const getWalStatusMeta = (status?: SlotWalStatus): WalStatusMeta => + WAL_STATUS_META[status ?? 'unknown'] + +// Legend entries from healthiest to most severe, ending with the unknown/unavailable case. +export const WAL_STATUS_LEGEND: WalStatusMeta[] = [ + WAL_STATUS_META.reserved, + WAL_STATUS_META.extended, + WAL_STATUS_META.unreserved, + WAL_STATUS_META.lost, + WAL_STATUS_META.unknown, +] + +export const getWalStatusSeverity = (status?: SlotWalStatus): LagSeverity => + getWalStatusMeta(status).severity + +// Slot-loss risk from how much of the slot's WAL budget has been consumed, rather than fixed byte +// thresholds: max_slot_wal_keep_size ≈ retained WAL (restart_lsn_bytes) + remaining headroom +// (safe_wal_size_bytes), so the consumed fraction is how close the slot is to the "lost" state. +// A null/absent safe_wal_size_bytes now means unlimited retention, so it carries no budget risk. +export const SLOT_LOSS_WARNING_RATIO = 0.75 +export const SLOT_LOSS_CRITICAL_RATIO = 0.9 + +export const getSlotBudgetSeverity = ( + retainedBytes?: number, + safeWalSizeBytes?: number | null +): LagSeverity => { + if ( + typeof retainedBytes !== 'number' || + typeof safeWalSizeBytes !== 'number' || + !Number.isFinite(retainedBytes) || + !Number.isFinite(safeWalSizeBytes) + ) { + return 'normal' + } + + const total = retainedBytes + safeWalSizeBytes + // Nothing retained and no headroom left: nothing to flag (also avoids a 0/0 division). When the + // headroom is 0 but WAL is still retained, the ratio is 1 and the slot is correctly critical. + if (total <= 0) return 'normal' + + const consumedRatio = retainedBytes / total + if (consumedRatio >= SLOT_LOSS_CRITICAL_RATIO) return 'critical' + if (consumedRatio >= SLOT_LOSS_WARNING_RATIO) return 'warning' + return 'normal' +} + +const SEVERITY_RANK: Record = { normal: 0, warning: 1, critical: 2 } + +const maxSeverity = (a: LagSeverity, b: LagSeverity): LagSeverity => + SEVERITY_RANK[a] >= SEVERITY_RANK[b] ? a : b + +// Overall slot health = the worse of the reported WAL status and how close the WAL budget is to +// running out. Used to color/flag the lag value in the destinations list. +export const getSlotHealthSeverity = (slot?: { + restart_lsn_bytes?: number + safe_wal_size_bytes?: number | null + wal_status?: SlotWalStatus +}): LagSeverity => { + if (!slot) return 'normal' + return maxSeverity( + getWalStatusSeverity(slot.wal_status), + getSlotBudgetSeverity(slot.restart_lsn_bytes, slot.safe_wal_size_bytes) + ) +} diff --git a/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/SlotLagMetrics.tsx b/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/SlotLagMetrics.tsx index 526981129965a..cede6f71a1da9 100644 --- a/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/SlotLagMetrics.tsx +++ b/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/SlotLagMetrics.tsx @@ -1,38 +1,64 @@ +import dayjs from 'dayjs' import { Info } from 'lucide-react' +import { type ReactNode } from 'react' import { Tooltip, TooltipContent, TooltipTrigger } from 'ui' import { SlotLagMetricKey, SlotLagMetrics } from './ReplicationPipelineStatus.types' import { getFormattedLagValue } from './ReplicationPipelineStatus.utils' +import { SlotConnectionIndicator, SlotStatusBadge } from './SlotStatus' -const SLOT_LAG_FIELDS: { +interface SlotLagField { key: SlotLagMetricKey label: string type: 'bytes' | 'duration' - description: string -}[] = [ + description: ReactNode + // Friendly label to show in place of a literal "0 bytes" when there's nothing to report. + zeroLabel?: string + // Friendly label to show when the value is null/absent (e.g. unlimited WAL retention). + nullLabel?: string + // Optional hover text for the value, derived from the raw value (e.g. an absolute timestamp). + getValueTooltip?: (value: number) => string +} + +const SLOT_LAG_FIELDS: SlotLagField[] = [ { key: 'confirmed_flush_lsn_bytes', - label: 'WAL Flush lag (size)', + label: 'Waiting to sync', type: 'bytes', - description: - 'Bytes between the newest WAL record applied locally and the latest flushed WAL record acknowledged by ETL.', - }, - { - key: 'flush_lag', - label: 'WAL Flush lag (time)', - type: 'duration', - description: - 'Time between flushing recent WAL locally and receiving notification that ETL has written and flushed it.', + description: "Changes in your database the pipeline hasn't synced yet.", + zeroLabel: 'Caught up', }, { key: 'safe_wal_size_bytes', - label: 'Remaining WAL size', + label: 'Room before pausing', type: 'bytes', - description: - 'Bytes still available to write to WAL before this slot risks entering the "lost" state.', + description: ( + <> + How much more can pile up before the pipeline has to be set up again. Controlled by the{' '} + max_slot_wal_keep_size setting. + + ), + nullLabel: 'Unlimited', + }, + { + key: 'reply_time_lag', + label: 'Last check-in', + type: 'duration', + description: 'Time since the pipeline last reported back to your database.', + zeroLabel: 'Just now', + // reply_time_lag is "milliseconds ago", so the absolute time is now minus that, in local time. + getValueTooltip: (ms) => dayjs().subtract(ms, 'millisecond').format('MMM D, YYYY, h:mm:ss A'), }, ] +// Resolves a field's value into a display string (+ optional precise detail), honoring the +// friendly zero/null labels before falling back to the formatted byte/duration value. +const getFieldDisplay = (field: SlotLagField, value: number | null | undefined) => { + if (value == null) return { display: field.nullLabel ?? 'n/a', detail: undefined } + if (field.zeroLabel && value === 0) return { display: field.zeroLabel, detail: undefined } + return getFormattedLagValue(field.type, value) +} + export const SlotLagMetricsInline = ({ tableName, metrics, @@ -45,14 +71,17 @@ export const SlotLagMetricsInline = ({ {tableName} - • + + + {metrics.wal_status && } +
- {SLOT_LAG_FIELDS.map(({ key, label, type }) => { - const { display } = getFormattedLagValue(type, metrics[key]) + {SLOT_LAG_FIELDS.map((field) => { + const { display } = getFieldDisplay(field, metrics[field.key]) return ( - + - {label} + {field.label} {display} @@ -87,40 +116,54 @@ export const SlotLagMetricsList = ({ return (
- {SLOT_LAG_FIELDS.map(({ key, label, type, description }) => ( -
-
- - {label} - {showMetricInfo && ( + {SLOT_LAG_FIELDS.map((field) => { + const rawValue = metrics[field.key] + const { display, detail } = getFieldDisplay(field, rawValue) + const valueTooltip = + field.getValueTooltip && typeof rawValue === 'number' + ? field.getValueTooltip(rawValue) + : undefined + return ( +
+
+ + {field.label} + {showMetricInfo && ( + + + + + + {field.description} + + + )} + +
+
+ {valueTooltip ? ( - + {display} - - {description} + + {valueTooltip} - )} - - - {(() => { - const { display, detail } = getFormattedLagValue(type, metrics[key]) - return ( -
+ ) : ( {display} - {detail && {detail}} -
- ) - })()} -
- ))} + )} + {detail && {detail}} + +
+ ) + })} ) } diff --git a/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/SlotStatus.tsx b/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/SlotStatus.tsx new file mode 100644 index 0000000000000..e1d8f0c24d7fe --- /dev/null +++ b/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/SlotStatus.tsx @@ -0,0 +1,135 @@ +import { Info } from 'lucide-react' +import { + Badge, + cn, + Popover, + PopoverContent, + PopoverTrigger, + Tooltip, + TooltipContent, + TooltipTrigger, +} from 'ui' + +import { SlotWalStatus } from './ReplicationPipelineStatus.types' +import { getWalStatusMeta, WAL_STATUS_LEGEND } from './ReplicationPipelineStatus.utils' +import { InlineLink } from '@/components/ui/InlineLink' +import { DOCS_URL } from '@/lib/constants' + +export type SlotStatusContext = 'pipeline' | 'table' + +const CONNECTION_TEXT: Record = { + pipeline: { + active: "This pipeline's replication slot is active and being used right now.", + inactive: "This pipeline's replication slot is not active right now.", + }, + table: { + active: "This table's replication slot is active and being used right now.", + inactive: "This table's replication slot is not active right now.", + }, +} + +/** + * Colored badge for a slot's WAL status, with the plain-language meaning on hover. + * Pass `context="table"` in the per-table inline view to show table-specific descriptions. + */ +export const SlotStatusBadge = ({ + status, + context = 'pipeline', +}: { + status?: SlotWalStatus + context?: SlotStatusContext +}) => { + const meta = getWalStatusMeta(status) + const description = context === 'table' ? meta.tableDescription : meta.description + return ( + + + + {meta.label} + + + + {description} + + + ) +} + +/** + * Info button opening a legend that explains every possible slot status. + */ +export const SlotStatusLegend = () => { + return ( + + + + + +
+

Slot statuses

+

+ How safely your database is keeping the changes the pipeline still needs. +

+
+
    + {WAL_STATUS_LEGEND.map((meta) => ( +
  • +
    + {meta.label} +
    + + {meta.description} + +
    • + ))} +
+
+ + Learn more about monitoring replication + +
+ + + ) +} + +/** + * Small dot + label indicating whether the slot has a live replication connection. + * Pass `context="table"` in the per-table inline view to show table-specific descriptions. + */ +export const SlotConnectionIndicator = ({ + isActive, + context = 'pipeline', +}: { + isActive?: boolean + context?: SlotStatusContext +}) => { + const text = CONNECTION_TEXT[context] + return ( + + + + + {isActive ? 'Connected' : 'Not connected'} + + + + {isActive ? text.active : text.inactive} + + + ) +} diff --git a/apps/studio/components/interfaces/Database/Replication/RowMenu.tsx b/apps/studio/components/interfaces/Database/Replication/RowMenu.tsx index 6360b277b7935..984ea9051a8f6 100644 --- a/apps/studio/components/interfaces/Database/Replication/RowMenu.tsx +++ b/apps/studio/components/interfaces/Database/Replication/RowMenu.tsx @@ -9,6 +9,10 @@ import { DropdownMenuItem, DropdownMenuSeparator, DropdownMenuTrigger, + Tooltip, + TooltipContent, + TooltipTrigger, + WarningIcon, } from 'ui' import { ShimmeringLoader } from 'ui-patterns/ShimmeringLoader' @@ -18,7 +22,6 @@ import { PIPELINE_ENABLE_ALLOWED_FROM, } from './Pipeline.utils' import { PipelineStatusName } from './Replication.constants' -import AlertError from '@/components/ui/AlertError' import { ReplicationPipelineStatusData } from '@/data/replication/pipeline-status-query' import { Pipeline } from '@/data/replication/pipelines-query' import { useRestartPipelineHelper } from '@/data/replication/restart-pipeline-helper' @@ -134,7 +137,18 @@ export const RowMenu = ({
{isLoading && } - {isError && } + {isError && ( + + + + + + + + Couldn't load status{error?.message ? `: ${error.message}` : '.'} + + + )} diff --git a/apps/studio/components/interfaces/Integrations/CronJobs/CronJobs.utils.test.ts b/apps/studio/components/interfaces/Integrations/CronJobs/CronJobs.utils.test.ts index f79435a6455b8..93c3b1c73902c 100644 --- a/apps/studio/components/interfaces/Integrations/CronJobs/CronJobs.utils.test.ts +++ b/apps/studio/components/interfaces/Integrations/CronJobs/CronJobs.utils.test.ts @@ -211,6 +211,116 @@ describe('parseCronJobCommand', () => { }) }) + it('should keep a jsonb_build_object header value that contains a comma intact', () => { + const command = `select net.http_post( url:='https://example.com/api/endpoint', headers:=jsonb_build_object('Accept', 'application/json, text/plain'), timeout_milliseconds:=5000 );` + expect(parseCronJobCommand(command, 'random_project_ref')).toStrictEqual({ + endpoint: 'https://example.com/api/endpoint', + method: 'POST', + httpHeaders: [{ name: 'Accept', value: 'application/json, text/plain' }], + httpBody: '', + timeoutMs: 5000, + type: 'http_request', + snippet: command, + }) + }) + + it('should not shift later jsonb_build_object headers when an earlier value contains a comma', () => { + const command = `select net.http_post( url:='https://example.com/api/endpoint', headers:=jsonb_build_object('Accept', 'application/json, text/plain', 'Authorization', 'Bearer abc'), timeout_milliseconds:=5000 );` + expect(parseCronJobCommand(command, 'random_project_ref')).toStrictEqual({ + endpoint: 'https://example.com/api/endpoint', + method: 'POST', + httpHeaders: [ + { name: 'Accept', value: 'application/json, text/plain' }, + { name: 'Authorization', value: 'Bearer abc' }, + ], + httpBody: '', + timeoutMs: 5000, + type: 'http_request', + snippet: command, + }) + }) + + it('should keep a jsonb_build_object header value that contains parentheses intact', () => { + const command = `select net.http_post( url:='https://example.com/api/endpoint', headers:=jsonb_build_object('User-Agent', 'Mozilla/5.0 (compatible)'), timeout_milliseconds:=5000 );` + expect(parseCronJobCommand(command, 'random_project_ref')).toStrictEqual({ + endpoint: 'https://example.com/api/endpoint', + method: 'POST', + httpHeaders: [{ name: 'User-Agent', value: 'Mozilla/5.0 (compatible)' }], + httpBody: '', + timeoutMs: 5000, + type: 'http_request', + snippet: command, + }) + }) + + it('should keep an escaped quote inside a comma-containing jsonb_build_object value', () => { + const command = `select net.http_post( url:='https://example.com/api/endpoint', headers:=jsonb_build_object('X-Company', 'O''Reilly, Inc'), timeout_milliseconds:=5000 );` + expect(parseCronJobCommand(command, 'random_project_ref')).toStrictEqual({ + endpoint: 'https://example.com/api/endpoint', + method: 'POST', + httpHeaders: [{ name: 'X-Company', value: "O'Reilly, Inc" }], + httpBody: '', + timeoutMs: 5000, + type: 'http_request', + snippet: command, + }) + }) + + it('should unescape backslashes in an E-prefixed jsonb_build_object header value', () => { + const command = `select net.http_post( url:='https://example.com/api/endpoint', headers:=jsonb_build_object('X-Custom', E'value\\\\here'), timeout_milliseconds:=5000 );` + expect(parseCronJobCommand(command, 'random_project_ref')).toStrictEqual({ + endpoint: 'https://example.com/api/endpoint', + method: 'POST', + httpHeaders: [{ name: 'X-Custom', value: 'value\\here' }], + httpBody: '', + timeoutMs: 5000, + type: 'http_request', + snippet: command, + }) + }) + + it('should keep later jsonb_build_object headers when an earlier value contains parentheses', () => { + const command = `select net.http_post( url:='https://example.com/api/endpoint', headers:=jsonb_build_object('User-Agent', 'Mozilla/5.0 (compatible)', 'Accept', 'application/json'), timeout_milliseconds:=5000 );` + expect(parseCronJobCommand(command, 'random_project_ref')).toStrictEqual({ + endpoint: 'https://example.com/api/endpoint', + method: 'POST', + httpHeaders: [ + { name: 'User-Agent', value: 'Mozilla/5.0 (compatible)' }, + { name: 'Accept', value: 'application/json' }, + ], + httpBody: '', + timeoutMs: 5000, + type: 'http_request', + snippet: command, + }) + }) + + it('should parse jsonb_build_object headers when there is whitespace before the opening parenthesis', () => { + const command = `select net.http_post( url:='https://example.com/api/endpoint', headers:=jsonb_build_object ('Accept', 'application/json'), timeout_milliseconds:=5000 );` + expect(parseCronJobCommand(command, 'random_project_ref')).toStrictEqual({ + endpoint: 'https://example.com/api/endpoint', + method: 'POST', + httpHeaders: [{ name: 'Accept', value: 'application/json' }], + httpBody: '', + timeoutMs: 5000, + type: 'http_request', + snippet: command, + }) + }) + + it('should not let a comma inside a jsonb_build_object value swallow the following body argument', () => { + const command = `select net.http_post( url:='https://example.com/api/endpoint', headers:=jsonb_build_object('Accept', 'application/json, text/plain'), body:='{"key": "value"}', timeout_milliseconds:=5000 );` + expect(parseCronJobCommand(command, 'random_project_ref')).toStrictEqual({ + endpoint: 'https://example.com/api/endpoint', + method: 'POST', + httpHeaders: [{ name: 'Accept', value: 'application/json, text/plain' }], + httpBody: '{"key": "value"}', + timeoutMs: 5000, + type: 'http_request', + snippet: command, + }) + }) + it('should return an HTTP request config with GET method and empty body', () => { const command = `select net.http_get( url:='https://example.com/api/endpoint', headers:=jsonb_build_object(), timeout_milliseconds:=5000 );` expect(parseCronJobCommand(command, 'random_project_ref')).toStrictEqual({ diff --git a/apps/studio/components/interfaces/Integrations/CronJobs/CronJobs.utils.tsx b/apps/studio/components/interfaces/Integrations/CronJobs/CronJobs.utils.tsx index ac68d4948f958..22884d07b7635 100644 --- a/apps/studio/components/interfaces/Integrations/CronJobs/CronJobs.utils.tsx +++ b/apps/studio/components/interfaces/Integrations/CronJobs/CronJobs.utils.tsx @@ -13,6 +13,76 @@ const unescapeSqlLiteral = (value = '', isEscapeString = false) => { return isEscapeString ? unescaped.replaceAll('\\\\', '\\') : unescaped } +/** + * Strips the surrounding quotes from a single SQL string literal and unescapes its + * contents, handling the optional `E''` escape-string prefix. + */ +const unwrapSqlLiteral = (token: string) => { + const trimmed = token.trim() + const isEscapeString = /^e'/i.test(trimmed) + const withoutPrefix = isEscapeString ? trimmed.slice(1) : trimmed + const withoutQuotes = withoutPrefix.replace(/^'|'$/g, '') + return unescapeSqlLiteral(withoutQuotes, isEscapeString) +} + +/** + * Splits the argument list of a `jsonb_build_object(...)` call into its individual + * values, honoring single-quoted SQL string literals (with '' escapes) and nested + * parentheses. A naive split on ',' corrupts a header name or value that legitimately + * contains a comma or parenthesis, which then gets persisted on save and no longer + * matches what the user entered. + */ +const parseJsonBuildObjectArgs = (command: string) => { + const match = command.match(/headers:=jsonb_build_object\s*\(/i) + if (!match || match.index === undefined) return [] + + const args: string[] = [] + let current = '' + let depth = 1 + let inQuote = false + let hasContent = false + + for (let i = match.index + match[0].length; i < command.length && depth > 0; i++) { + const char = command[i] + + if (inQuote) { + if (char === "'" && command[i + 1] === "'") { + current += "''" + i++ + continue + } + if (char === "'") inQuote = false + current += char + continue + } + + if (char === "'") { + inQuote = true + current += char + hasContent = true + } else if (char === '(') { + depth++ + current += char + } else if (char === ')') { + depth-- + if (depth > 0) current += char + } else if (char === ',' && depth === 1) { + args.push(current) + current = '' + } else { + current += char + if (char.trim().length > 0) hasContent = true + } + } + + // Unbalanced parentheses: bail rather than emit mangled fragments. + if (depth !== 0) return [] + + if (hasContent || args.length > 0) args.push(current) + + return args.map(unwrapSqlLiteral) +} + export function buildCronCreateQuery( name: string, schedule: string, @@ -75,18 +145,14 @@ export const parseCronJobCommand = (originalCommand: string, projectRef: string) const timeoutMatch = command.match(/timeout_milliseconds:=(\d+)/i) const timeout = timeoutMatch?.[1] || '' - const headersJsonBuildObjectMatch = command.match(/headers:=jsonb_build_object\(([^)]*)/i) - const headersJsonBuildObject = headersJsonBuildObjectMatch?.[1] || '' - let headersObjs: { name: string; value: string }[] = [] - if (headersJsonBuildObject) { - const headers = headersJsonBuildObject - .split(',') - .map((s) => unescapeSqlLiteral(s.trim().replace(/^'|'$/g, ''))) - - for (let i = 0; i < headers.length; i += 2) { - if (headers[i] && headers[i].length > 0) { - headersObjs.push({ name: headers[i], value: headers[i + 1] }) + if (/headers:=jsonb_build_object\s*\(/i.test(command)) { + const args = parseJsonBuildObjectArgs(command) + + for (let i = 0; i < args.length; i += 2) { + const name = args[i] + if (name && name.length > 0) { + headersObjs.push({ name, value: args[i + 1] ?? '' }) } } } else { diff --git a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/InputField.tsx b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/InputField.tsx index 1843f4937df6c..39467a3ea3cfb 100644 --- a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/InputField.tsx +++ b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/InputField.tsx @@ -173,7 +173,7 @@ export const InputField = ({ } if (includes(TEXT_TYPES, field.format)) { - const isTruncated = isValueTruncated(field.value) + const isTruncated = isValueTruncated(field.value, field.format) /** * Handle `undefined` as the default value of the input field @@ -250,7 +250,7 @@ export const InputField = ({ } if (includes(JSON_TYPES, field.format)) { - const isTruncated = isValueTruncated(field.value) + const isTruncated = isValueTruncated(field.value, field.format) return ( candidate.name === column + )?.format + const isTruncated = isValueTruncated(jsonString, columnFormat) const { mutate: getCellValue, isPending, isSuccess, reset } = useGetCellValueMutation() diff --git a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.tsx b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.tsx index 198ffdef1731b..ced4a6bc70a11 100644 --- a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.tsx +++ b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.tsx @@ -133,7 +133,7 @@ export const RowEditor = ({ updateEditorDirty() const payload = isNewRecord - ? generateRowObjectFromFields({ fields: rowFields }) + ? generateRowObjectFromFields({ fields: rowFields, useDefaultForEmptyValues: true }) : generateUpdateRowPayload(row, rowFields) const configuration = { identifiers: {}, rowIdx: -1 } diff --git a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.utils.test.ts b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.utils.test.ts index d6e7bdd4a9ad6..4735666c5abf4 100644 --- a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.utils.test.ts +++ b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.utils.test.ts @@ -176,6 +176,51 @@ describe('generateRowObjectFromFields', () => { const result = generateRowObjectFromFields({ fields: sampleRowFields }) expect(result).toEqual({ name: '' }) }) + it('should omit cleared identity and default fields for new rows', () => { + const sampleRowFields: RowField[] = [ + { + id: '1', + name: 'id', + value: '', + comment: '', + defaultValue: null, + format: 'int8', + enums: [], + isNullable: false, + isIdentity: true, + isPrimaryKey: true, + }, + { + id: '2', + name: 'created_at', + value: '', + comment: '', + defaultValue: 'now()', + format: 'timestamptz', + enums: [], + isNullable: false, + isIdentity: false, + isPrimaryKey: false, + }, + { + id: '3', + name: 'name', + value: '', + comment: '', + defaultValue: null, + format: 'text', + enums: [], + isNullable: false, + isIdentity: false, + isPrimaryKey: false, + }, + ] + const result = generateRowObjectFromFields({ + fields: sampleRowFields, + useDefaultForEmptyValues: true, + }) + expect(result).toEqual({ name: '' }) + }) it('should discern NULL values for text', () => { const sampleRowFields: RowField[] = [ { @@ -274,12 +319,12 @@ describe('isValueTruncated', () => { // Pattern 4: Multi-dimensional array (lines 211-212) // column[1:50]::type[] - no special marker, just detect by [[ pattern - expect(isValueTruncated('[["item"]]')).toBe(true) - expect(isValueTruncated('[["item1","item2"]]')).toBe(true) + expect(isValueTruncated('[["item"]]', '_text')).toBe(true) + expect(isValueTruncated('[["item1","item2"]]', '_text')).toBe(true) }) it('should detect multidimensional arrays', () => { - expect(isValueTruncated('[["item1", "item2"]]')).toBe(true) + expect(isValueTruncated('[["item1", "item2"]]', '_text')).toBe(true) }) it('should detect truncated JSON arrays with truncated flag', () => { @@ -303,6 +348,12 @@ describe('isValueTruncated', () => { expect(isValueTruncated({} as any)).toBe(false) }) + it('should not flag small jsonb arrays as truncated', () => { + expect( + isValueTruncated('[["infrequently","frequently","constantly"],["90","30","180"]]', 'jsonb') + ).toBe(false) + }) + it('should test edge cases that could break coordination with table-row-query.ts', () => { // Test values that are just under/at the thresholds to ensure boundaries are correct diff --git a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.utils.ts b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.utils.ts index 0d9da5a66acbc..d9e83aae4253d 100644 --- a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.utils.ts +++ b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.utils.ts @@ -97,7 +97,7 @@ export const validateFields = (fields: RowField[]) => { } } if (field.format.includes('json') && (field.value?.length ?? 0) > 0) { - const isTruncated = isValueTruncated(field.value) + const isTruncated = isValueTruncated(field.value, field.format) // don't validate if the value is truncated if (isTruncated) return @@ -191,16 +191,25 @@ const convertInputDatetimeToPostgresDatetime = ( export const generateRowObjectFromFields = ({ fields, includeUndefinedValues = false, + useDefaultForEmptyValues = false, }: { fields: RowField[] includeUndefinedValues?: boolean + useDefaultForEmptyValues?: boolean }): object => { const rowObject = {} as any fields.forEach((field) => { const isArray = field.format.startsWith('_') const value = field.value + const shouldUseDefaultValue = + useDefaultForEmptyValues && + value === '' && + (field.isIdentity || field.defaultValue !== null) && + !TEXT_TYPES.includes(field.format) - if (isArray && value !== null) { + if (shouldUseDefaultValue) { + rowObject[field.name] = undefined + } else if (isArray && value !== null) { rowObject[field.name] = tryParseJson(value) } else if (field.format.includes('json')) { if (typeof field.value === 'object') { @@ -244,7 +253,7 @@ export const generateUpdateRowPayload = (originalRow: any, fields: RowField[]) = } else if (type !== undefined && JSON_TYPES.includes(type)) { // don't update if the value is truncated. This is to enable the user to change cell values on rows which have // truncated JSON values. If the user - const isTruncated = isValueTruncated(field?.value) + const isTruncated = isValueTruncated(field?.value, field?.format) if (!isTruncated) { payload[property] = rowObject[property] } @@ -263,7 +272,9 @@ export const generateUpdateRowPayload = (originalRow: any, fields: RowField[]) = /** * Checks if the value is truncated. The JSON types are usually truncated if they're too big to show in the editor. */ -export const isValueTruncated = (value: string | null | undefined) => { +export const isValueTruncated = (value: string | null | undefined, format?: string | null) => { + const isArrayColumn = typeof format === 'string' && format.startsWith('_') + return ( (typeof value === 'string' && value.endsWith('...') && value.length > MAX_CHARACTERS) || // if the value is an array which total representation is > MAX_CHARACTERS @@ -275,9 +286,7 @@ export const isValueTruncated = (value: string | null | undefined) => { // If the array have MAX_ARRAY_SIZE elements in it // its a large truncated array (value.match(/","/g) || []).length === MAX_ARRAY_SIZE) || - // if the string represent a multi-dimentional array we always consider it as possibly truncated - // so user load the whole value before edition - (typeof value === 'string' && value.startsWith('[["')) || + (typeof value === 'string' && isArrayColumn && value.startsWith('[["')) || // [Joshen] For json arrays, refer to getTableRowsSql from table-row-query // for array types, we're adding {"truncated": true} as the last item of the JSON to // maintain the JSON array structure diff --git a/apps/studio/components/layouts/ProjectLayout/index.test.tsx b/apps/studio/components/layouts/ProjectLayout/index.test.tsx index adc6ee2d47c62..8fca42fbcc86b 100644 --- a/apps/studio/components/layouts/ProjectLayout/index.test.tsx +++ b/apps/studio/components/layouts/ProjectLayout/index.test.tsx @@ -79,6 +79,7 @@ vi.mock('common', () => ({ `free-micro-upgrade-banner-dismissed-${ref}`, PROJECT_INTEGRATION_BANNER_DISMISSED: (ref: string, integrationSource: string) => `project-integration-banner-dismissed-${ref}-${integrationSource}`, + UNIFIED_LOGS_BANNER_DISMISSED: 'unified-logs-banner-dismissed', }, isFeatureEnabled: () => false, })) @@ -170,7 +171,10 @@ vi.mock('@/hooks/misc/useLocalStorage', () => ({ })) vi.mock('@/components/ui/BannerStack/BannerStackProvider', () => ({ - BANNER_ID: { FREE_MICRO_UPGRADE: 'free-micro-upgrade-banner' }, + BANNER_ID: { + FREE_MICRO_UPGRADE: 'free-micro-upgrade-banner', + UNIFIED_LOGS: 'unified-logs-banner', + }, useBannerStack: () => ({ addBanner: mockAddBanner, dismissBanner: mockDismissBanner, @@ -182,6 +186,20 @@ vi.mock('@/components/ui/BannerStack/Banners/BannerFreeMicroUpgrade', () => ({ BannerFreeMicroUpgrade: () => null, })) +vi.mock('@/components/ui/BannerStack/Banners/BannerUnifiedLogs', () => ({ + BannerUnifiedLogs: () => null, +})) + +vi.mock('@/components/interfaces/App/FeaturePreview/FeaturePreviewContext', () => ({ + useUnifiedLogsPreview: () => ({ + isEnabled: false, + isEligible: false, + isLoading: false, + enable: () => {}, + disable: () => {}, + }), +})) + vi.mock('@/data/usage/resource-warnings-query', () => ({ useResourceWarningsQuery: () => ({ data: mockResourceWarningsState.current }), })) diff --git a/apps/studio/components/layouts/ProjectLayout/index.tsx b/apps/studio/components/layouts/ProjectLayout/index.tsx index 7e831ada5f19c..f7de0dd72ad6f 100644 --- a/apps/studio/components/layouts/ProjectLayout/index.tsx +++ b/apps/studio/components/layouts/ProjectLayout/index.tsx @@ -41,9 +41,11 @@ import { RestoreFailedState } from './RestoreFailedState' import { RestoringState } from './RestoringState' import { UnhealthyState } from './UnhealthyState' import { UpgradingState } from './UpgradingState' +import { useUnifiedLogsPreview } from '@/components/interfaces/App/FeaturePreview/FeaturePreviewContext' import { CreateBranchModal } from '@/components/interfaces/BranchManagement/CreateBranchModal' import { ProjectAPIDocs } from '@/components/interfaces/ProjectAPIDocs/ProjectAPIDocs' import { BannerFreeMicroUpgrade } from '@/components/ui/BannerStack/Banners/BannerFreeMicroUpgrade' +import { BannerUnifiedLogs } from '@/components/ui/BannerStack/Banners/BannerUnifiedLogs' import { BANNER_ID, useBannerStack } from '@/components/ui/BannerStack/BannerStackProvider' import { ButtonTooltip } from '@/components/ui/ButtonTooltip' import PartnerIcon from '@/components/ui/PartnerIcon' @@ -155,6 +157,11 @@ export const ProjectLayout = forwardRef { + if (!selectedProject?.ref) return + if (showUnifiedLogsBanner && !isUnifiedLogsBannerDismissed) { + addBanner({ + id: BANNER_ID.UNIFIED_LOGS, + isDismissed: false, + content: , + priority: 1, + }) + } else { + dismissBanner(BANNER_ID.UNIFIED_LOGS) + } + }, [ + selectedProject?.ref, + showUnifiedLogsBanner, + isUnifiedLogsBannerDismissed, + addBanner, + dismissBanner, + ]) + useLayoutEffect(() => { const unregister = registerOpenMenu(() => { setMobileSheetContent( diff --git a/apps/studio/components/ui/BannerStack/BannerStack.tsx b/apps/studio/components/ui/BannerStack/BannerStack.tsx index 664f8e40f792c..28420784fe678 100644 --- a/apps/studio/components/ui/BannerStack/BannerStack.tsx +++ b/apps/studio/components/ui/BannerStack/BannerStack.tsx @@ -1,76 +1,78 @@ -import { AnimatePresence, motion } from 'framer-motion' +import { AnimatePresence, motion, useReducedMotion } from 'framer-motion' import { useState } from 'react' import { useBannerStack } from './BannerStackProvider' +const PEEK_OFFSET = 8 +const MAX_PEEKS = 2 + +const SPRING = { type: 'spring', stiffness: 300, damping: 30 } as const + export const BannerStack = () => { const { banners } = useBannerStack() const [isHovered, setIsHovered] = useState(false) + const reduceMotion = useReducedMotion() const activeBanners = banners.filter((b) => !b.isDismissed) + if (activeBanners.length === 0) return null - const PEEK_HEIGHT = 4 - const CARD_GAP = 4 - const CARD_HEIGHT = 212 + const [frontBanner, ...extraBanners] = activeBanners + const peekCount = Math.min(extraBanners.length, MAX_PEEKS) + // Deepest sliver first so the closer ones paint on top of it. + const peeks = Array.from({ length: peekCount }, (_, i) => peekCount - i) - if (activeBanners.length === 0) return null + const transition = reduceMotion ? { duration: 0 } : SPRING return ( setIsHovered(true)} onMouseLeave={() => setIsHovered(false)} - animate={{ - y: isHovered ? -8 : 0, - }} - transition={{ - type: 'spring', - stiffness: 300, - damping: 25, - }} + animate={{ y: isHovered ? -8 : 0 }} + transition={transition} > -
- - {activeBanners.map((banner, index) => { - const isBottomBanner = index === 0 - const reverseIndex = activeBanners.length - 1 - index - const collapsedY = index * PEEK_HEIGHT - const expandedY = index * (CARD_HEIGHT + CARD_GAP) - - return ( +
+ + {!isHovered && + peeks.map((depth) => ( setIsHovered(true)} - onMouseLeave={() => setIsHovered(false)} - style={{ - position: isBottomBanner ? 'relative' : 'absolute', - bottom: isBottomBanner ? undefined : 0, - right: isBottomBanner ? undefined : 0, - zIndex: 30 + reverseIndex, - transformOrigin: 'center bottom', - }} - className="w-full max-w-72" - > - {banner.content} - - ) - })} + key={`peek-${depth}`} + className="absolute inset-0 rounded-2xl border bg-surface-75 shadow-lg" + style={{ transformOrigin: 'center top' }} + initial={{ opacity: 0 }} + animate={{ opacity: 1, y: -depth * PEEK_OFFSET, scaleX: 1 - depth * 0.06 }} + exit={{ opacity: 0, y: 0, scaleX: 1 }} + transition={transition} + /> + ))} + + + {frontBanner.content} +
+ + + {isHovered && + extraBanners.map((banner, index) => ( + + {banner.content} + + ))} + ) } diff --git a/apps/studio/components/ui/BannerStack/BannerStackProvider.tsx b/apps/studio/components/ui/BannerStack/BannerStackProvider.tsx index eef631513f0b4..b8aa2e24b7aff 100644 --- a/apps/studio/components/ui/BannerStack/BannerStackProvider.tsx +++ b/apps/studio/components/ui/BannerStack/BannerStackProvider.tsx @@ -8,6 +8,7 @@ export const BANNER_ID = { RLS_TESTER: 'rls-tester-banner', FREE_MICRO_UPGRADE: 'free-micro-upgrade-banner', TOS_UPDATE: 'tos-update-banner', + UNIFIED_LOGS: 'unified-logs-banner', } as const export type BannerId = (typeof BANNER_ID)[keyof typeof BANNER_ID] diff --git a/apps/studio/components/ui/BannerStack/Banners/BannerUnifiedLogs.tsx b/apps/studio/components/ui/BannerStack/Banners/BannerUnifiedLogs.tsx new file mode 100644 index 0000000000000..d8317e6b268fe --- /dev/null +++ b/apps/studio/components/ui/BannerStack/Banners/BannerUnifiedLogs.tsx @@ -0,0 +1,76 @@ +import { LOCAL_STORAGE_KEYS } from 'common' +import { useParams } from 'common/hooks' +import Link from 'next/link' +import { Badge, Button } from 'ui' + +import { BannerCard } from '../BannerCard' +import { useBannerStack } from '../BannerStackProvider' +import { UnifiedLogsCarousel } from './UnifiedLogsCarousel' +import { + useFeaturePreviewModal, + useUnifiedLogsPreview, +} from '@/components/interfaces/App/FeaturePreview/FeaturePreviewContext' +import { useLocalStorageQuery } from '@/hooks/misc/useLocalStorage' +import { useTrack } from '@/lib/telemetry/track' + +export const BannerUnifiedLogs = () => { + const { ref } = useParams() + const track = useTrack() + const { dismissBanner } = useBannerStack() + const { isEnabled } = useUnifiedLogsPreview() + const { selectFeaturePreview } = useFeaturePreviewModal() + const [, setIsDismissed] = useLocalStorageQuery( + LOCAL_STORAGE_KEYS.UNIFIED_LOGS_BANNER_DISMISSED, + false + ) + + return ( + { + setIsDismissed(true) + dismissBanner('unified-logs-banner') + track('unified_logs_banner_dismiss_button_clicked') + }} + > +
+
+ + Beta + +
+ +
+
+
+

Unified Logs is here

+

+ Search and correlate logs across all of your services from a single place. +

+
+
+ {isEnabled ? ( + + ) : ( + + )} +
+
+ + ) +} diff --git a/apps/studio/components/ui/BannerStack/Banners/UnifiedLogsCarousel.tsx b/apps/studio/components/ui/BannerStack/Banners/UnifiedLogsCarousel.tsx new file mode 100644 index 0000000000000..55958c3fb49b4 --- /dev/null +++ b/apps/studio/components/ui/BannerStack/Banners/UnifiedLogsCarousel.tsx @@ -0,0 +1,95 @@ +import dayjs from 'dayjs' +import { AnimatePresence, motion } from 'framer-motion' +import { useEffect, useRef, useState } from 'react' +import { cn } from 'ui' + +// Number of rows visible in the carousel viewport. We keep one extra row in +// state so the row sliding out of view has something to animate towards. +const VISIBLE_ROWS = 3 +const ROW_HEIGHT = 28 +const TICK_MS = 3000 + +const SAMPLES = [200, 200, 201, 200, 304, 400, 200, 500] as const + +interface LogEntry { + id: number + timestamp: string + status: number +} + +const makeEntry = (id: number, offsetSeconds = 0): LogEntry => ({ + id, + timestamp: dayjs().subtract(offsetSeconds, 'second').format('DD MMM YY HH:mm:ss'), + status: SAMPLES[id % SAMPLES.length], +}) + +const LogRow = ({ entry }: { entry: LogEntry }) => { + const isDestructive = entry.status >= 500 + const isWarning = entry.status >= 400 && entry.status < 500 + return ( +
+ + + {entry.timestamp} + + + {entry.status} + +
+ ) +} + +export const UnifiedLogsCarousel = () => { + const counter = useRef(VISIBLE_ROWS) + const [logs, setLogs] = useState(() => + Array.from({ length: VISIBLE_ROWS + 1 }, (_, i) => makeEntry(VISIBLE_ROWS - i, i)) + ) + + useEffect(() => { + const interval = setInterval(() => { + counter.current += 1 + const next = makeEntry(counter.current) + setLogs((prev) => [next, ...prev].slice(0, VISIBLE_ROWS + 1)) + }, TICK_MS) + return () => clearInterval(interval) + }, []) + + return ( +
+ + {logs.map((entry, index) => ( + + + + ))} + +
+ ) +} diff --git a/apps/studio/pages/_app.tsx b/apps/studio/pages/_app.tsx index 33d4d6b42fe34..f55fde22c31ad 100644 --- a/apps/studio/pages/_app.tsx +++ b/apps/studio/pages/_app.tsx @@ -186,7 +186,7 @@ function CustomApp({ Component, pageProps }: AppPropsWithLayout) { {appTitle ?? 'Supabase'} - + {/* [Alaister]: This has to be an inline style tag here and not a separate component due to next/font */}