diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ae24bb..49cab89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -117,6 +117,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `PATCH /v1/software/{id}/analysis` and `PATCH /v1/catalogs/{id}/analysis` answer `500` on SQLite. Namespaces may now nest at most 64 objects and arrays, and deeper ones get `422` on both databases. +- An analysis number outside the PostgreSQL `numeric` range, such as + `1e131072`, made the analysis `PATCH` answer `500` on PostgreSQL. + Numbers must now fit a float64, with an exponent between -16383 and + 16383 and at most 16383 digits after the decimal point, and others get + `422` on both databases. - The analysis of a software or catalog grew without limit when each `PATCH` added a new namespace. A merged document larger than 1 MiB is now rejected with `422` and nothing is written. Patches that only remove diff --git a/analysis_numbers_test.go b/analysis_numbers_test.go new file mode 100644 index 0000000..6f674ef --- /dev/null +++ b/analysis_numbers_test.go @@ -0,0 +1,72 @@ +package main + +import ( + "encoding/json" + "net/http" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestAnalysisNumbersOutsideStorableRange checks that numbers PostgreSQL +// can't store as numeric are refused with 422 on both dialects, instead of +// failing the write with 500 on PostgreSQL only. +func TestAnalysisNumbersOutsideStorableRange(t *testing.T) { + for _, resource := range []struct{ table, id, title string }{ + {"software", rootSoftwareID, "can't update Software analysis"}, + {"catalogs", swissID, "can't update Catalog analysis"}, + } { + t.Run(resource.table, func(t *testing.T) { + loadFixtures(t) + + patch := func(number string) (int, map[string]any) { + t.Helper() + + body := `{"ns":{"v":1,"x":` + number + `}}` + req, err := newTestRequest(http.MethodPatch, "/v1/"+resource.table+"/"+resource.id+"/analysis", + strings.NewReader(body)) + require.NoError(t, err) + req.Header.Set("Authorization", goodToken) + req.Header.Set("Content-Type", "application/merge-patch+json") + + res, err := app.Test(req, -1) + require.NoError(t, err) + + defer res.Body.Close() + + var result map[string]any + require.NoError(t, json.NewDecoder(res.Body).Decode(&result)) + + return res.StatusCode, result + } + + for number, detail := range map[string]string{ + "1e131072": "numbers must fit a float64 without rounding to zero or infinity", + "-1e1000000": "numbers must fit a float64 without rounding to zero or infinity", + "1e-20000": "numbers must fit a float64 without rounding to zero or infinity", + "1" + strings.Repeat("0", 140000): "numbers must fit a float64 without rounding to zero or infinity", + "0e-20000": "numbers must have an exponent between -16383 and 16383", + "1." + strings.Repeat("0", 20000): "numbers must not have more than 16383 digits after the decimal point", + } { + code, result := patch(number) + + assert.Equal(t, http.StatusUnprocessableEntity, code, number[:min(len(number), 16)]) + assert.Equal(t, resource.title, result["title"]) + assert.Equal(t, "analysis.ns: "+detail, result["detail"]) + } + + code, result := patch("1.7976931348623157e308") + require.Equal(t, http.StatusOK, code, result) + + // The merged document holds only this namespace, so none of the + // refused patches was stored. + assert.Len(t, result, 1) + + namespace, ok := result["ns"].(map[string]any) + require.True(t, ok, result) + assert.InDelta(t, 1.7976931348623157e308, namespace["x"], 0) + }) + } +} diff --git a/internal/common/analysis.go b/internal/common/analysis.go index d0276b5..dfebb2e 100644 --- a/internal/common/analysis.go +++ b/internal/common/analysis.go @@ -6,6 +6,9 @@ import ( "encoding/json" "errors" "fmt" + "math" + "strconv" + "strings" "time" jsonpatch "github.com/evanphx/json-patch/v5" @@ -19,8 +22,19 @@ var ( errAnalysisUnexpectedDB = errors.New("unexpected database value type for analysis") errAnalysisAddTimestamp = errors.New("could not add the 't' timestamp") errAnalysisTooDeep = fmt.Errorf("must not nest objects and arrays deeper than %d levels", MaxAnalysisDepth) + errAnalysisNumberRange = errors.New("numbers must fit a float64 without rounding to zero or infinity") + errAnalysisNumberScale = fmt.Errorf("numbers must not have more than %d digits after the decimal point", + maxAnalysisNumberScale) + errAnalysisExponent = fmt.Errorf("numbers must have an exponent between -%d and %d", + maxAnalysisNumberScale, maxAnalysisNumberScale) ) +// maxAnalysisNumberScale is how many digits after the decimal point a +// number may have. PostgreSQL stores jsonb numbers as numeric, which +// refuses more than this, so 1.000... with 20000 zeros fits a float64 +// but can't be stored. +const maxAnalysisNumberScale = 16383 + // MaxAnalysisDepth is how many objects and arrays a namespace may nest, // the namespace object itself included. SQLite refuses JSON nested deeper // than 1000 levels, so a namespace must stay well below that to be merged @@ -162,33 +176,85 @@ func WithTimestamps(analysis AnalysisData, now time.Time) (AnalysisData, error) return result, nil } -// jsonDepth answers how deeply objects and arrays nest in data, 0 for a -// scalar. On invalid JSON it answers the depth reached before the error. -func jsonDepth(data []byte) int { +// checkValues walks data and answers errAnalysisTooDeep when objects and +// arrays nest deeper than MaxAnalysisDepth, or the checkNumber error of +// the first number both dialects can't store and read back. On invalid +// JSON it stops at the error and answers nil, leaving the report to +// json.Unmarshal. +func checkValues(data []byte) error { decoder := json.NewDecoder(bytes.NewReader(data)) - depth, maxDepth := 0, 0 + decoder.UseNumber() + + depth := 0 for { token, err := decoder.Token() if err != nil { - return maxDepth + // io.EOF ends a valid document, checkNamespace reports the + // rest through json.Unmarshal. + return nil //nolint:nilerr } - switch token { - case json.Delim('{'), json.Delim('['): - depth++ - maxDepth = max(maxDepth, depth) - case json.Delim('}'), json.Delim(']'): - depth-- + switch token := token.(type) { + case json.Delim: + if token == '{' || token == '[' { + depth++ + if depth > MaxAnalysisDepth { + return errAnalysisTooDeep + } + } else { + depth-- + } + case json.Number: + if err := checkNumber(string(token)); err != nil { + return err + } } } } +// checkNumber answers an error for a JSON number literal PostgreSQL can't +// store as numeric or a float64 can't hold: a number that rounds to +// infinity, or to zero while having a nonzero digit, an exponent beyond +// maxAnalysisNumberScale either way, or more than maxAnalysisNumberScale +// digits after the decimal point once the exponent is applied, as in 1.0 +// followed by 20000 zeros. Most JSON clients read numbers into a float64. +func checkNumber(literal string) error { + value, err := strconv.ParseFloat(literal, 64) + if err != nil || math.IsInf(value, 0) { + return errAnalysisNumberRange + } + + mantissa, exponent, _ := strings.Cut(strings.ToLower(literal), "e") + _, fraction, _ := strings.Cut(mantissa, ".") + + if value == 0 && strings.ContainsAny(mantissa, "123456789") { + return errAnalysisNumberRange + } + + // Zero takes any exponent as a float64, 0e-20000 included, but + // PostgreSQL keeps the scale the exponent implies. + exp := 0 + if exponent != "" { + exp, err = strconv.Atoi(exponent) + if err != nil || exp < -maxAnalysisNumberScale || exp > maxAnalysisNumberScale { + return errAnalysisExponent + } + } + + if len(fraction)-exp > maxAnalysisNumberScale { + return errAnalysisNumberScale + } + + return nil +} + // checkNamespace checks that raw is a JSON object with an integer "v" -// between 1 and 999, nesting at most MaxAnalysisDepth levels. +// between 1 and 999, nesting at most MaxAnalysisDepth levels and holding +// only numbers checkNumber accepts. func checkNamespace(raw json.RawMessage) error { - if jsonDepth(raw) > MaxAnalysisDepth { - return errAnalysisTooDeep + if err := checkValues(raw); err != nil { + return err } var meta namespaceMeta diff --git a/internal/common/analysis_test.go b/internal/common/analysis_test.go index 8731968..5594a8e 100644 --- a/internal/common/analysis_test.go +++ b/internal/common/analysis_test.go @@ -2,6 +2,7 @@ package common import ( "encoding/json" + "fmt" "strings" "testing" "time" @@ -177,6 +178,85 @@ func TestWithTimestamps(t *testing.T) { }) } +func TestCheckNumber(t *testing.T) { + zeros := strings.Repeat("0", maxAnalysisNumberScale) + + for _, tc := range []struct { + literal string + want error + }{ + {"0", nil}, + {"-0", nil}, + {"42", nil}, + {"-1.5", nil}, + {"1E+308", nil}, + {"1.7976931348623157e308", nil}, + {"4.9e-324", nil}, + {"0e16383", nil}, + {"0e-16383", nil}, + {"0." + zeros, nil}, + {"1e400", errAnalysisNumberRange}, + {"1e131072", errAnalysisNumberRange}, + {"-1e1000000", errAnalysisNumberRange}, + {"1" + strings.Repeat("0", 140000), errAnalysisNumberRange}, + {"1e-400", errAnalysisNumberRange}, + {"1e-20000", errAnalysisNumberRange}, + {"0." + zeros + "1", errAnalysisNumberRange}, + {"0e-20000", errAnalysisExponent}, + {"0e20000", errAnalysisExponent}, + {"0e99999999999999999999", errAnalysisExponent}, + {"0e-99999999999999999999", errAnalysisExponent}, + {"1." + zeros + "0", errAnalysisNumberScale}, + {"0." + zeros + "0", errAnalysisNumberScale}, + {"1." + zeros + "e-1", errAnalysisNumberScale}, + {"0.0e-16383", errAnalysisNumberScale}, + } { + name := tc.literal + if len(name) > 32 { + name = fmt.Sprintf("%s... (%d bytes)", name[:16], len(name)) + } + + t.Run(name, func(t *testing.T) { + assert.Equal(t, tc.want, checkNumber(tc.literal)) + }) + } +} + +func TestWithTimestampsNumbers(t *testing.T) { + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + + t.Run("refuses a number nested in an array", func(t *testing.T) { + input := AnalysisData{ + "scanner": json.RawMessage(`{"v": 1, "x": [{"y": 1e400}]}`), + } + + _, err := WithTimestamps(input, now) + + assert.EqualError(t, err, + "analysis.scanner: numbers must fit a float64 without rounding to zero or infinity") + }) + + t.Run("refuses a number as v", func(t *testing.T) { + input := AnalysisData{ + "scanner": json.RawMessage(`{"v": 1e-20000}`), + } + + _, err := WithTimestamps(input, now) + + assert.ErrorIs(t, err, errAnalysisNumberRange) + }) + + t.Run("keeps numbers inside strings", func(t *testing.T) { + input := AnalysisData{ + "scanner": json.RawMessage(`{"v": 1, "x": "1e400"}`), + } + + _, err := WithTimestamps(input, now) + + assert.NoError(t, err) + }) +} + func TestCheckAnalysisSize(t *testing.T) { payload := func(size int) json.RawMessage { return json.RawMessage(`{"v":1,"x":"` + strings.Repeat("a", size) + `"}`) diff --git a/open-catalog-api.oas.yaml b/open-catalog-api.oas.yaml index 80486f3..b260fdd 100644 --- a/open-catalog-api.oas.yaml +++ b/open-catalog-api.oas.yaml @@ -395,6 +395,9 @@ paths: a JSON object with an integer "v" field. Objects and arrays may nest at most 64 levels in a namespace, the namespace object included. A deeper namespace is rejected with 422. + Numbers must fit a float64 without rounding to zero or infinity, + with an exponent between -16383 and 16383 and at most 16383 digits + after the decimal point. Other numbers are rejected with 422. The merged document may take at most 1048576 bytes (1 MiB) as compact JSON. A patch that would make it larger is rejected with 422 and nothing is written. A patch that only removes namespaces is @@ -1562,6 +1565,9 @@ paths: a JSON object with an integer "v" field. Objects and arrays may nest at most 64 levels in a namespace, the namespace object included. A deeper namespace is rejected with 422. + Numbers must fit a float64 without rounding to zero or infinity, + with an exponent between -16383 and 16383 and at most 16383 digits + after the decimal point. Other numbers are rejected with 422. The merged document may take at most 1048576 bytes (1 MiB) as compact JSON. A patch that would make it larger is rejected with 422 and nothing is written. A patch that only removes namespaces is