diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 94b84f20f66..e6564dd2f84 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -24,7 +24,7 @@ "ghcr.io/jungaretti/features/make:1": {}, "ghcr.io/devcontainers/features/docker-in-docker:2.12.0": {}, "ghcr.io/devcontainers/features/python:1": { - "version": "3.9" + "version": "3.11" }, "ghcr.io/devcontainers-extra/features/poetry:2": {}, "ghcr.io/devcontainers/features/node:1": { diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index b30d7a75b63..2b2d977a5f8 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -24,7 +24,7 @@ services: - default command: sleep infinity mysql: - image: mysql:8 + image: mysql:8.4.10 volumes: - mysql-storage:/var/lib/mysql restart: always diff --git a/.github/workflows/config-ui.yml b/.github/workflows/config-ui.yml index fcf52a19e25..3d56075b280 100644 --- a/.github/workflows/config-ui.yml +++ b/.github/workflows/config-ui.yml @@ -29,7 +29,7 @@ jobs: - name: Set up Node.js uses: actions/setup-node@v6 with: - node-version: '18' + node-version: '24' cache: 'yarn' cache-dependency-path: config-ui/yarn.lock - name: Install with Yarn diff --git a/Makefile b/Makefile index 5e18c277968..8639ce2dc6f 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ build-config-ui-image: cd config-ui; docker build -t $(IMAGE_REPO)/devlake-config-ui:$(TAG) --file ./Dockerfile . build-grafana-image: - cd grafana; docker build -t $(IMAGE_REPO)/devlake-dashboard:$(TAG) --file ./backend/Dockerfile . + cd grafana; docker build -t $(IMAGE_REPO)/devlake-dashboard:$(TAG) --file ./Dockerfile . build-images: build-server-image build-config-ui-image build-grafana-image diff --git a/backend/.dockerignore b/backend/.dockerignore new file mode 100644 index 00000000000..36ee77aae07 --- /dev/null +++ b/backend/.dockerignore @@ -0,0 +1,24 @@ +# Build artifacts: regenerated inside the image (GOBIN=/app/bin + make build-plugin/build-server). +# Shipping the host's pre-built bin/ (multiple GB of plugin .so files) bloats the +# build context and the builder layer, which can exhaust the Docker VM and crash BuildKit. +bin/ + +# Generated mocks (regenerated via make mock inside the image) +mocks/ + +# Runtime logs +logs/ + +# Python build/venv artifacts (regenerated; Dockerfile.server reinstalls from +# requirements.txt / pyproject and runs python/build.sh). Source under python/ is kept. +**/.venv/ +**/__pycache__/ +**/*.pyc +python/.devlake-python-build-root/ + +# Test / coverage output +*.out +coverage.txt + +# OS / editor junk +.DS_Store diff --git a/backend/Dockerfile b/backend/Dockerfile index 4246ae5f9f5..87e9adaefd8 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -50,7 +50,7 @@ RUN if [ "$(arch)" != "x86_64" ] ; then \ fi RUN go install github.com/vektra/mockery/v2@v2.53.6 -RUN go install github.com/swaggo/swag/cmd/swag@v1.16.1 +RUN go install github.com/swaggo/swag/cmd/swag@v1.16.6 COPY --from=debian-amd64 /usr/include /rootfs-amd64/usr/include COPY --from=debian-amd64 /usr/lib/x86_64-linux-gnu /rootfs-amd64/usr/lib/x86_64-linux-gnu @@ -113,7 +113,7 @@ RUN cd /usr/local/deps/target/lib && \ done -FROM python:3.9-slim-bookworm as base +FROM python:3.11-slim-bookworm as base RUN apt-get update && \ apt-get install -y python3-dev python3-pip tar pkg-config curl libssh2-1 zlib1g libffi-dev default-libmysqlclient-dev libpq-dev tini git openssh-client corkscrew && \ diff --git a/backend/Dockerfile.local b/backend/Dockerfile.local index ee306dd7e7a..2fadbcec90e 100644 --- a/backend/Dockerfile.local +++ b/backend/Dockerfile.local @@ -48,7 +48,7 @@ RUN mkdir -p /tmp/build && cd /tmp/build && \ ldconfig RUN go install github.com/vektra/mockery/v2@v2.53.6 -RUN go install github.com/swaggo/swag/cmd/swag@v1.16.1 +RUN go install github.com/swaggo/swag/cmd/swag@v1.16.6 WORKDIR /app COPY . /app diff --git a/backend/Makefile b/backend/Makefile index 369e1490f27..7aca0a2e471 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -28,7 +28,7 @@ all: build go-dep: go install github.com/vektra/mockery/v2@v2.53.6 - go install github.com/swaggo/swag/cmd/swag@v1.16.1 + go install github.com/swaggo/swag/cmd/swag@v1.16.6 go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2 go-dev-tools: diff --git a/backend/core/models/domainlayer/codequality/cq_issue_code_blocks.go b/backend/core/models/domainlayer/codequality/cq_issue_code_blocks.go index 294bc5d3cb2..c522fc5cce0 100644 --- a/backend/core/models/domainlayer/codequality/cq_issue_code_blocks.go +++ b/backend/core/models/domainlayer/codequality/cq_issue_code_blocks.go @@ -22,7 +22,7 @@ import "github.com/apache/incubator-devlake/core/models/domainlayer" type CqIssueCodeBlock struct { domainlayer.DomainEntity IssueKey string `json:"key" gorm:"index"` - Component string `gorm:"index"` + Component string `gorm:"type:text"` StartLine int EndLine int StartOffset int diff --git a/backend/core/models/domainlayer/codequality/cq_project_metrics_history.go b/backend/core/models/domainlayer/codequality/cq_project_metrics_history.go new file mode 100644 index 00000000000..90f64ce660c --- /dev/null +++ b/backend/core/models/domainlayer/codequality/cq_project_metrics_history.go @@ -0,0 +1,46 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package codequality + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/domainlayer" +) + +type CqProjectMetricsHistory struct { + domainlayer.DomainEntity + ProjectKey string `gorm:"index;type:varchar(500)"` + AnalysisDate time.Time `gorm:"index"` + Coverage *float64 + Ncloc *int + Bugs *int + ReliabilityRating string `gorm:"type:varchar(5)"` + CodeSmells *int + SqaleRating string `gorm:"type:varchar(5)"` + Complexity *int + CognitiveComplexity *int + Vulnerabilities *int + SecurityRating string `gorm:"type:varchar(5)"` + SecurityHotspots *int + DuplicatedLinesDensity *float64 +} + +func (CqProjectMetricsHistory) TableName() string { + return "cq_project_metrics_history" +} diff --git a/backend/core/models/domainlayer/domaininfo/domaininfo.go b/backend/core/models/domainlayer/domaininfo/domaininfo.go index b2cc5431fb2..b88289e8d8f 100644 --- a/backend/core/models/domainlayer/domaininfo/domaininfo.go +++ b/backend/core/models/domainlayer/domaininfo/domaininfo.go @@ -56,6 +56,7 @@ func GetDomainTablesInfo() []dal.Tabler { &codequality.CqIssue{}, &codequality.CqIssueImpact{}, &codequality.CqProject{}, + &codequality.CqProjectMetricsHistory{}, // crossdomain &crossdomain.Account{}, &crossdomain.BoardRepo{}, diff --git a/backend/core/models/domainlayer/ticket/sprint.go b/backend/core/models/domainlayer/ticket/sprint.go index 29c449aca8f..47d1c2b35df 100644 --- a/backend/core/models/domainlayer/ticket/sprint.go +++ b/backend/core/models/domainlayer/ticket/sprint.go @@ -18,9 +18,10 @@ limitations under the License. package ticket import ( + "time" + "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/models/domainlayer" - "time" ) var ( @@ -31,13 +32,15 @@ var ( type Sprint struct { domainlayer.DomainEntity - Name string `gorm:"type:varchar(255)"` - Url string `gorm:"type:varchar(255)"` - Status string `gorm:"type:varchar(100)"` - StartedDate *time.Time - EndedDate *time.Time - CompletedDate *time.Time - OriginalBoardID string `gorm:"type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + Url string `gorm:"type:varchar(255)"` + Status string `gorm:"type:varchar(100)"` + StartedDate *time.Time + EndedDate *time.Time + CompletedDate *time.Time + OriginalBoardID string `gorm:"type:varchar(255)"` + CommittedStoryPoint *float64 + CompletedStoryPoint *float64 } func (Sprint) TableName() string { diff --git a/backend/core/models/migrationscripts/20260629_change_issue_component_to_text.go b/backend/core/models/migrationscripts/20260629_change_issue_component_to_text.go new file mode 100644 index 00000000000..94d965d6295 --- /dev/null +++ b/backend/core/models/migrationscripts/20260629_change_issue_component_to_text.go @@ -0,0 +1,41 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*changeIssueComponentToText)(nil) + +type changeIssueComponentToText struct{} + +func (*changeIssueComponentToText) Up(basicRes context.BasicRes) errors.Error { + // The 20240813 migration targeted the non-existent plural "components" column. + return basicRes.GetDal().ModifyColumnType("issues", "component", "text") +} + +func (*changeIssueComponentToText) Version() uint64 { + return 20260629120000 +} + +func (*changeIssueComponentToText) Name() string { + return "change issues.component type to text" +} diff --git a/backend/core/models/migrationscripts/20260629_change_issue_component_to_text_test.go b/backend/core/models/migrationscripts/20260629_change_issue_component_to_text_test.go new file mode 100644 index 00000000000..482c42b5001 --- /dev/null +++ b/backend/core/models/migrationscripts/20260629_change_issue_component_to_text_test.go @@ -0,0 +1,82 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" +) + +type modifyColumnTypeCall struct { + tableName string + columnName string + columnType string +} + +type recordingDal struct { + dal.Dal + call *modifyColumnTypeCall +} + +func (d *recordingDal) ModifyColumnType(tableName string, columnName string, columnType string) errors.Error { + d.call = &modifyColumnTypeCall{tableName, columnName, columnType} + return nil +} + +type basicResWithDal struct { + context.BasicRes + database dal.Dal +} + +func (r *basicResWithDal) GetDal() dal.Dal { + return r.database +} + +func TestChangeIssueComponentToText(t *testing.T) { + database := new(recordingDal) + script := new(changeIssueComponentToText) + + if err := script.Up(&basicResWithDal{database: database}); err != nil { + t.Fatalf("migration failed: %v", err) + } + + want := &modifyColumnTypeCall{"issues", "component", "text"} + if database.call == nil || *database.call != *want { + t.Fatalf("ModifyColumnType call = %#v, want %#v", database.call, want) + } + if script.Version() != 20260629120000 { + t.Fatalf("Version() = %d, want 20260629120000", script.Version()) + } + if script.Name() != "change issues.component type to text" { + t.Fatalf("Name() = %q, want %q", script.Name(), "change issues.component type to text") + } + + found := false + for _, registeredScript := range All() { + if registeredScript.Version() == script.Version() { + found = true + break + } + } + if !found { + t.Fatalf("migration version %d is not registered", script.Version()) + } +} diff --git a/backend/core/models/migrationscripts/20260701_change_cq_issue_code_blocks_component_to_text.go b/backend/core/models/migrationscripts/20260701_change_cq_issue_code_blocks_component_to_text.go new file mode 100644 index 00000000000..b84d1d9ce1b --- /dev/null +++ b/backend/core/models/migrationscripts/20260701_change_cq_issue_code_blocks_component_to_text.go @@ -0,0 +1,44 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*changeCqIssueCodeBlocksComponentToText)(nil) + +type changeCqIssueCodeBlocksComponentToText struct{} + +func (script *changeCqIssueCodeBlocksComponentToText) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + if err := db.DropIndexes("cq_issue_code_blocks", "idx_cq_issue_code_blocks_component"); err != nil { + return err + } + return db.ModifyColumnType("cq_issue_code_blocks", "component", "text") +} + +func (*changeCqIssueCodeBlocksComponentToText) Version() uint64 { + return 20260701000000 +} + +func (*changeCqIssueCodeBlocksComponentToText) Name() string { + return "change cq_issue_code_blocks.component type to text" +} diff --git a/backend/core/models/migrationscripts/20260707_add_cq_project_metrics_history.go b/backend/core/models/migrationscripts/20260707_add_cq_project_metrics_history.go new file mode 100644 index 00000000000..de66ae82914 --- /dev/null +++ b/backend/core/models/migrationscripts/20260707_add_cq_project_metrics_history.go @@ -0,0 +1,42 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type addCqProjectMetricsHistory struct{} + +func (u *addCqProjectMetricsHistory) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &archived.CqProjectMetricsHistory{}, + ) +} + +func (*addCqProjectMetricsHistory) Version() uint64 { + return 20260707153201 +} + +func (*addCqProjectMetricsHistory) Name() string { + return "add cq_project_metrics_history domain table" +} diff --git a/backend/core/models/migrationscripts/20260722_add_sprint_velocity_fields.go b/backend/core/models/migrationscripts/20260722_add_sprint_velocity_fields.go new file mode 100644 index 00000000000..369b70b035d --- /dev/null +++ b/backend/core/models/migrationscripts/20260722_add_sprint_velocity_fields.go @@ -0,0 +1,49 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*addSprintVelocityFields)(nil) + +type sprint20260722 struct { + CommittedStoryPoint *float64 + CompletedStoryPoint *float64 +} + +func (sprint20260722) TableName() string { + return "sprints" +} + +type addSprintVelocityFields struct{} + +func (script *addSprintVelocityFields) Up(basicRes context.BasicRes) errors.Error { + return basicRes.GetDal().AutoMigrate(new(sprint20260722)) +} + +func (*addSprintVelocityFields) Version() uint64 { + return 20260722100000 +} + +func (*addSprintVelocityFields) Name() string { + return "add committed_story_point/completed_story_point to sprints" +} diff --git a/backend/core/models/migrationscripts/archived/base.go b/backend/core/models/migrationscripts/archived/base.go index 7af72cd65d8..1ece6543821 100644 --- a/backend/core/models/migrationscripts/archived/base.go +++ b/backend/core/models/migrationscripts/archived/base.go @@ -19,8 +19,6 @@ package archived import ( "time" - - "golang.org/x/exp/constraints" ) type DomainEntity struct { @@ -44,7 +42,14 @@ type ScopeConfig struct { Entities []string `gorm:"type:json;serializer:json" json:"entities" mapstructure:"entities"` } -type GenericModel[T string | constraints.Unsigned] struct { +// unsignedInteger matches all unsigned integer types, replacing +// golang.org/x/exp/constraints.Unsigned to avoid a transitive dependency +// on that module which requires Go 1.23+ in recent versions. +type unsignedInteger interface { + ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 | ~uintptr +} + +type GenericModel[T string | unsignedInteger] struct { ID T `gorm:"primaryKey" json:"id"` CreatedAt time.Time `json:"createdAt"` UpdatedAt time.Time `json:"updatedAt"` diff --git a/backend/core/models/migrationscripts/archived/cq_project_metrics_history.go b/backend/core/models/migrationscripts/archived/cq_project_metrics_history.go new file mode 100644 index 00000000000..934627368e2 --- /dev/null +++ b/backend/core/models/migrationscripts/archived/cq_project_metrics_history.go @@ -0,0 +1,42 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import "time" + +type CqProjectMetricsHistory struct { + DomainEntity + ProjectKey string `gorm:"index;type:varchar(500)"` + AnalysisDate time.Time `gorm:"index"` + Coverage *float64 + Ncloc *int + Bugs *int + ReliabilityRating string `gorm:"type:varchar(5)"` + CodeSmells *int + SqaleRating string `gorm:"type:varchar(5)"` + Complexity *int + CognitiveComplexity *int + Vulnerabilities *int + SecurityRating string `gorm:"type:varchar(5)"` + SecurityHotspots *int + DuplicatedLinesDensity *float64 +} + +func (CqProjectMetricsHistory) TableName() string { + return "cq_project_metrics_history" +} diff --git a/backend/core/models/migrationscripts/register.go b/backend/core/models/migrationscripts/register.go index b4596154e2c..9dd5e762b4a 100644 --- a/backend/core/models/migrationscripts/register.go +++ b/backend/core/models/migrationscripts/register.go @@ -145,5 +145,9 @@ func All() []plugin.MigrationScript { new(modifyCicdDeploymentsToText), new(increaseCqIssuesProjectKeyLength), new(addAuthSessions), + new(changeIssueComponentToText), + new(changeCqIssueCodeBlocksComponentToText), + new(addCqProjectMetricsHistory), + new(addSprintVelocityFields), } } diff --git a/backend/helpers/e2ehelper/migration_db.go b/backend/helpers/e2ehelper/migration_db.go new file mode 100644 index 00000000000..3d7fe2d49e5 --- /dev/null +++ b/backend/helpers/e2ehelper/migration_db.go @@ -0,0 +1,126 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2ehelper + +import ( + "fmt" + "net/url" + "strings" + "testing" + + "github.com/apache/incubator-devlake/core/config" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/impls/dalgorm" + "github.com/apache/incubator-devlake/impls/logruslog" + "gorm.io/gorm" +) + +// NewIsolatedMigrationDb creates a dedicated, empty database next to the one +// referenced by E2E_DB_URL (`_`) and returns a connection to it. +// +// Tests that execute the REAL migration scripts must not share the regular e2e +// database: the other plugin e2e tests seed/AutoMigrate tables (domain layer +// included) without recording anything in `_devlake_migration_history`, so a +// subsequent migration run fails with errors such as +// "Table 'cicd_pipeline_commits' already exists" when it tries to create or +// rename a table that is already there. +// +// The database is dropped again when the test finishes. If E2E_DB_URL is not +// set the test is skipped. +func NewIsolatedMigrationDb(t *testing.T, suffix string) *gorm.DB { + cfg := config.GetConfig() + e2eDbUrl := cfg.GetString("E2E_DB_URL") + if e2eDbUrl == "" { + t.Skip("E2E_DB_URL is not set; skipping migration schema check") + } + u, err := url.Parse(e2eDbUrl) + if err != nil { + t.Fatalf("unable to parse E2E_DB_URL: %v", err) + } + isolatedName := fmt.Sprintf("%s_%s", strings.TrimPrefix(u.Path, "/"), suffix) + quotedName := quoteDbName(u.Scheme, isolatedName) + + gormConf := &gorm.Config{SkipDefaultTransaction: true} + adminDb, err := runner.MakeDbConnection(e2eDbUrl, gormConf) + if err != nil { + t.Fatalf("unable to connect to E2E_DB_URL: %v", err) + } + if err = adminDb.Exec("DROP DATABASE IF EXISTS " + quotedName).Error; err != nil { + t.Fatalf("unable to drop leftover database %s: %v", isolatedName, err) + } + if err = adminDb.Exec("CREATE DATABASE " + quotedName).Error; err != nil { + t.Fatalf("unable to create database %s: %v", isolatedName, err) + } + closeDb(adminDb) + + isolatedUrl := *u + isolatedUrl.Path = "/" + isolatedName + db, err := runner.MakeDbConnection(isolatedUrl.String(), gormConf) + if err != nil { + t.Fatalf("unable to connect to %s: %v", isolatedName, err) + } + + // migration scripts and models read DB_URL from the global config, keep it + // consistent with the connection we hand out and restore it afterwards. + previousDbUrl := cfg.GetString("DB_URL") + cfg.Set("DB_URL", isolatedUrl.String()) + + // Some migration scripts refuse to run without an encryption secret + // (e.g. jira 20220716: "jira v0.11 invalid encKey"). CI does not + // necessarily provide one, so fall back to a deterministic test value. + // dalgorm.Init registers the `encdec` GORM serializer used by connection + // models - without it migrations fail with "invalid serializer type encdec" + // (runner.CreateBasicRes does not register it, only CreateAppBasicRes does). + if cfg.GetString(plugin.EncodeKeyEnvStr) == "" { + cfg.Set(plugin.EncodeKeyEnvStr, "devlake-e2e-test-encryption-secret") + } + dalgorm.Init(cfg.GetString(plugin.EncodeKeyEnvStr)) + + t.Cleanup(func() { + cfg.Set("DB_URL", previousDbUrl) + closeDb(db) + cleanupDb, cleanupErr := runner.MakeDbConnection(e2eDbUrl, gormConf) + if cleanupErr != nil { + t.Logf("unable to connect for dropping %s: %v", isolatedName, cleanupErr) + return + } + defer closeDb(cleanupDb) + if dropErr := cleanupDb.Exec("DROP DATABASE IF EXISTS " + quotedName).Error; dropErr != nil { + t.Logf("unable to drop database %s: %v", isolatedName, dropErr) + } + }) + + logruslog.Global.Info("running migrations against isolated database %s", isolatedName) + return db +} + +func quoteDbName(scheme string, name string) string { + // database names are derived from E2E_DB_URL + a constant suffix, but quote + // them anyway to stay safe with reserved words. + if strings.EqualFold(scheme, "mysql") { + return "`" + strings.ReplaceAll(name, "`", "") + "`" + } + return `"` + strings.ReplaceAll(name, `"`, "") + `"` +} + +func closeDb(db *gorm.DB) { + if sqlDb, err := db.DB(); err == nil { + _ = sqlDb.Close() + } +} diff --git a/backend/helpers/oidchelper/authorization.go b/backend/helpers/oidchelper/authorization.go new file mode 100644 index 00000000000..c740666a275 --- /dev/null +++ b/backend/helpers/oidchelper/authorization.go @@ -0,0 +1,42 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oidchelper + +import "strings" + +func (c *Config) IsUserAllowed(email string) bool { + if len(c.AllowEmails) == 0 && + len(c.AllowDomains) == 0 { + return true + } + + email = strings.ToLower(strings.TrimSpace(email)) + + if _, ok := c.AllowEmails[email]; ok { + return true + } + + _, domain, ok := strings.Cut(email, "@") + if ok { + if _, ok := c.AllowDomains[domain]; ok { + return true + } + } + + return false +} diff --git a/backend/helpers/oidchelper/authorization_test.go b/backend/helpers/oidchelper/authorization_test.go new file mode 100644 index 00000000000..e73a32e92d3 --- /dev/null +++ b/backend/helpers/oidchelper/authorization_test.go @@ -0,0 +1,119 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oidchelper + +import "testing" + +func TestIsUserAllowed(t *testing.T) { + cases := []struct { + name string + cfg Config + email string + want bool + }{ + { + name: "no restrictions", + cfg: Config{}, + email: "user@example.com", + want: true, + }, + { + name: "allowed email", + cfg: Config{ + AllowEmails: map[string]struct{}{ + "user@example.com": {}, + }, + }, + email: "user@example.com", + want: true, + }, + { + name: "blocked email", + cfg: Config{ + AllowEmails: map[string]struct{}{ + "user@example.com": {}, + }, + }, + email: "other@example.com", + want: false, + }, + { + name: "allowed domain", + cfg: Config{ + AllowDomains: map[string]struct{}{ + "example.com": {}, + }, + }, + email: "user@example.com", + want: true, + }, + { + name: "blocked domain", + cfg: Config{ + AllowDomains: map[string]struct{}{ + "example.com": {}, + }, + }, + email: "user@other.com", + want: false, + }, + { + name: "email case insensitive", + cfg: Config{ + AllowEmails: map[string]struct{}{ + "user@example.com": {}, + }, + }, + email: "USER@example.com", + want: true, + }, + { + name: "domain case insensitive", + cfg: Config{ + AllowDomains: map[string]struct{}{ + "example.com": {}, + }, + }, + email: "user@EXAMPLE.COM", + want: true, + }, + { + name: "invalid email", + cfg: Config{ + AllowDomains: map[string]struct{}{ + "example.com": {}, + }, + }, + email: "not-an-email", + want: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := tc.cfg.IsUserAllowed(tc.email); got != tc.want { + t.Errorf( + "IsUserAllowed(%q) = %v, want %v", + tc.email, + got, + tc.want, + ) + } + }) + } +} diff --git a/backend/helpers/oidchelper/config.go b/backend/helpers/oidchelper/config.go index a23606df1d4..9d1b8ae47c2 100644 --- a/backend/helpers/oidchelper/config.go +++ b/backend/helpers/oidchelper/config.go @@ -67,6 +67,11 @@ type Config struct { Providers map[string]*ProviderConfig LogoutRedirect bool + // Optional OIDC authorization restrictions. + // Empty means no restriction. + AllowEmails map[string]struct{} + AllowDomains map[string]struct{} + SessionSecret []byte SessionTTL time.Duration @@ -136,6 +141,8 @@ func LoadConfig(basicRes context.BasicRes) (*Config, error) { SessionTTL: ttl, CookieDomain: strings.TrimSpace(cfg.GetString("COOKIE_DOMAIN")), CookieSecure: cookieSecure, + AllowEmails: parseStringSet(cfg.GetString("OIDC_ALLOW_EMAILS")), + AllowDomains: parseStringSet(cfg.GetString("OIDC_ALLOW_DOMAINS")), } if !out.OIDCEnabled { @@ -205,6 +212,23 @@ func parseProviderNames(raw string) []string { seen[n] = struct{}{} out = append(out, n) } + + return out +} + +func parseStringSet(raw string) map[string]struct{} { + out := make(map[string]struct{}) + + for _, v := range strings.Split(raw, ",") { + v = strings.ToLower(strings.TrimSpace(v)) + + if v == "" { + continue + } + + out[v] = struct{}{} + } + return out } diff --git a/backend/impls/dalgorm/dalgorm.go b/backend/impls/dalgorm/dalgorm.go index ba635355f41..0445995fd50 100644 --- a/backend/impls/dalgorm/dalgorm.go +++ b/backend/impls/dalgorm/dalgorm.go @@ -453,6 +453,9 @@ func (d *Dalgorm) RenameTable(oldName, newName string) errors.Error { // DropIndexes drops indexes for specified table func (d *Dalgorm) DropIndexes(table string, indexNames ...string) errors.Error { for _, indexName := range indexNames { + if !d.db.Migrator().HasIndex(table, indexName) { + continue + } err := d.db.Migrator().DropIndex(table, indexName) if err != nil { return d.convertGormError(err) diff --git a/backend/plugins/azuredevops_go/tasks/shared.go b/backend/plugins/azuredevops_go/tasks/shared.go index e6fe0eca1e2..318850077e8 100644 --- a/backend/plugins/azuredevops_go/tasks/shared.go +++ b/backend/plugins/azuredevops_go/tasks/shared.go @@ -153,8 +153,11 @@ func change203To401(res *http.Response) errors.Error { // that failed due to a YAML syntax error never produce a usable timeline), instead // of aborting the entire subtask. func ignoreInvalidTimelineResponse(res *http.Response) errors.Error { - // Keep existing behaviour: treat 404 as a graceful skip (build was deleted). - if res.StatusCode == http.StatusNotFound { + // Treat 404 (build deleted) and 204 (build has no timeline data) as + // graceful skips so the subtask continues instead of failing. + // The 204 guard must come before the body is read because a 204 response + // has an empty body by definition and would cause the JSON parser to fail. + if res.StatusCode == http.StatusNotFound || res.StatusCode == http.StatusNoContent { return api.ErrIgnoreAndContinue } diff --git a/backend/plugins/azuredevops_go/tasks/shared_test.go b/backend/plugins/azuredevops_go/tasks/shared_test.go index 353ae3f4c91..d46e01d6c0d 100644 --- a/backend/plugins/azuredevops_go/tasks/shared_test.go +++ b/backend/plugins/azuredevops_go/tasks/shared_test.go @@ -41,6 +41,12 @@ func TestIgnoreInvalidTimelineResponse_404(t *testing.T) { assert.Equal(t, api.ErrIgnoreAndContinue, err, "404 should return ErrIgnoreAndContinue") } +func TestIgnoreInvalidTimelineResponse_204(t *testing.T) { + res := makeResponse(http.StatusNoContent, "") + err := ignoreInvalidTimelineResponse(res) + assert.Equal(t, api.ErrIgnoreAndContinue, err, "204 No Content should return ErrIgnoreAndContinue") +} + func TestIgnoreInvalidTimelineResponse_EmptyBody(t *testing.T) { res := makeResponse(http.StatusOK, "") err := ignoreInvalidTimelineResponse(res) diff --git a/backend/plugins/bitbucket/api/remote_api.go b/backend/plugins/bitbucket/api/remote_api.go index 211901ec3c6..36fa45a0de8 100644 --- a/backend/plugins/bitbucket/api/remote_api.go +++ b/backend/plugins/bitbucket/api/remote_api.go @@ -67,11 +67,15 @@ func listBitbucketWorkspaces( err errors.Error, ) { var res *http.Response + // /user/permissions/workspaces was removed and /workspaces deprecated by + // Bitbucket CHANGE-2770; /user/workspaces lists the current user's workspaces + // and is the supported replacement. res, err = apiClient.Get( - "/user/permissions/workspaces", + "/user/workspaces", url.Values{ - "sort": {"workspace.slug"}, - "fields": {"values.workspace.slug,values.workspace.name,pagelen,page,size"}, + // No sort/fields: /user/workspaces rejects sort=workspace.slug with + // HTTP 400 "Invalid field name". The bare call returns the nested + // workspace objects we need; WorkspaceResponse ignores extra fields. "page": {fmt.Sprintf("%v", page.Page)}, "pagelen": {fmt.Sprintf("%v", page.PageLen)}, }, @@ -98,11 +102,17 @@ func listBitbucketWorkspaces( for _, r := range resBody.Values { children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.BitbucketRepo]{ Type: api.RAS_ENTRY_TYPE_GROUP, - Id: r.Workspace.Slug, - Name: r.Workspace.Name, - FullName: r.Workspace.Name, + Id: r.GroupId(), + Name: r.GroupName(), + FullName: r.GroupName(), }) } + if resBody.Next != "" { + nextPage = &BitbucketRemotePagination{ + Page: page.Page + 1, + PageLen: page.PageLen, + } + } return } @@ -119,7 +129,7 @@ func listBitbucketRepos( var res *http.Response // list projects part res, err = apiClient.Get(fmt.Sprintf("/repositories/%s", workspace), url.Values{ - "fields": {"values.name,values.full_name,values.language,values.description,values.owner.display_name,values.created_on,values.updated_on,values.links.clone,values.links.html,pagelen,page,size"}, + "fields": {"values.name,values.full_name,values.language,values.description,values.owner.display_name,values.created_on,values.updated_on,values.links.clone,values.links.html,pagelen,page,size,next"}, "page": {fmt.Sprintf("%v", page.Page)}, "pagelen": {fmt.Sprintf("%v", page.PageLen)}, }, nil) @@ -149,9 +159,19 @@ func listBitbucketRepos( Data: r.ConvertApiScope(), }) } + if resBody.Next != "" { + nextPage = &BitbucketRemotePagination{ + Page: page.Page + 1, + PageLen: page.PageLen, + } + } return } +// searchBitbucketRepos searches repositories by name across the user's workspaces. +// The cross-workspace GET /repositories?role=member was removed by Bitbucket +// CHANGE-2770, so we enumerate workspaces and query the workspace-scoped +// GET /repositories/{workspace} endpoint for each, aggregating up to PageSize hits. func searchBitbucketRepos( apiClient plugin.ApiClient, params *dsmodels.DsRemoteApiScopeSearchParams, @@ -159,39 +179,84 @@ func searchBitbucketRepos( children []dsmodels.DsRemoteApiScopeListEntry[models.BitbucketRepo], err errors.Error, ) { - var res *http.Response - res, err = apiClient.Get( - "/repositories", - url.Values{ - "sort": {"name"}, - "fields": {"values.name,values.full_name,values.language,values.description,values.owner.display_name,values.created_on,values.updated_on,values.links.clone,values.links.html,pagelen,page,size"}, - "role": {"member"}, - "q": {fmt.Sprintf(`full_name~"%s"`, params.Search)}, - "page": {fmt.Sprintf("%v", params.Page)}, - "pagelen": {fmt.Sprintf("%v", params.PageSize)}, - }, - nil, - ) - if err != nil { - return nil, err + pageSize := params.PageSize + if pageSize == 0 { + pageSize = 100 } - var resBody models.ReposResponse - err = api.UnmarshalResponse(res, &resBody) + + workspaces, err := listAllBitbucketWorkspaces(apiClient) if err != nil { - return + return nil, err } - for _, r := range resBody.Values { - children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.BitbucketRepo]{ - Type: api.RAS_ENTRY_TYPE_SCOPE, - Id: r.FullName, - Name: r.Name, - FullName: r.FullName, - Data: r.ConvertApiScope(), - }) + + for _, workspace := range workspaces { + if len(children) >= pageSize { + break + } + var res *http.Response + res, err = apiClient.Get( + fmt.Sprintf("/repositories/%s", workspace), + url.Values{ + "sort": {"name"}, + "fields": {"values.name,values.full_name,values.language,values.description,values.owner.display_name,values.created_on,values.updated_on,values.links.clone,values.links.html,pagelen,page,size"}, + "q": {fmt.Sprintf(`name~"%s"`, params.Search)}, + "pagelen": {fmt.Sprintf("%v", pageSize)}, + }, + nil, + ) + if err != nil { + return nil, err + } + var resBody models.ReposResponse + err = api.UnmarshalResponse(res, &resBody) + if err != nil { + return + } + for _, r := range resBody.Values { + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.BitbucketRepo]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + Id: r.FullName, + Name: r.Name, + FullName: r.FullName, + Data: r.ConvertApiScope(), + }) + } } return } +// listAllBitbucketWorkspaces returns every workspace slug accessible to the +// authenticated user, following pagination of GET /2.0/user/workspaces. +func listAllBitbucketWorkspaces(apiClient plugin.ApiClient) ([]string, errors.Error) { + var slugs []string + for page := 1; ; page++ { + res, err := apiClient.Get( + "/user/workspaces", + url.Values{ + // No sort/fields (see listBitbucketWorkspaces): /user/workspaces + // returns 400 on sort=workspace.slug. + "page": {fmt.Sprintf("%v", page)}, + "pagelen": {"100"}, + }, + nil, + ) + if err != nil { + return nil, err + } + var resBody models.WorkspaceResponse + if err = api.UnmarshalResponse(res, &resBody); err != nil { + return nil, err + } + for _, w := range resBody.Values { + slugs = append(slugs, w.GroupId()) + } + if len(resBody.Values) == 0 || page*resBody.Pagelen >= resBody.Size { + break + } + } + return slugs, nil +} + // RemoteScopes list all available scopes on the remote server // @Summary list all available scopes on the remote server // @Description list all available scopes on the remote server diff --git a/backend/plugins/bitbucket/api/remote_api_test.go b/backend/plugins/bitbucket/api/remote_api_test.go new file mode 100644 index 00000000000..26bdbc4a2a4 --- /dev/null +++ b/backend/plugins/bitbucket/api/remote_api_test.go @@ -0,0 +1,83 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "io" + "net/http" + "net/url" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +type fakeApiClient struct{ body string } + +func (f *fakeApiClient) SetData(name string, data interface{}) {} +func (f *fakeApiClient) GetData(name string) interface{} { return nil } +func (f *fakeApiClient) SetHeaders(headers map[string]string) {} +func (f *fakeApiClient) SetBeforeFunction(callback plugin.ApiClientBeforeRequest) {} +func (f *fakeApiClient) GetBeforeFunction() plugin.ApiClientBeforeRequest { return nil } +func (f *fakeApiClient) SetAfterFunction(callback plugin.ApiClientAfterResponse) {} +func (f *fakeApiClient) GetAfterFunction() plugin.ApiClientAfterResponse { return nil } + +func (f *fakeApiClient) Get(path string, query url.Values, headers http.Header) (*http.Response, errors.Error) { + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(f.body))}, nil +} +func (f *fakeApiClient) Post(path string, query url.Values, body interface{}, headers http.Header) (*http.Response, errors.Error) { + return nil, nil +} + +func TestListBitbucketRepos_ReturnsNextPage(t *testing.T) { + client := &fakeApiClient{body: `{"pagelen":2,"page":1,"size":4, + "next":"https://api.bitbucket.org/2.0/repositories/myworkspace?page=2", + "values":[{"name":"repo-a","full_name":"myworkspace/repo-a"}, + {"name":"repo-b","full_name":"myworkspace/repo-b"}]}`} + children, nextPage, err := listBitbucketRepos(client, "myworkspace", BitbucketRemotePagination{Page: 1, PageLen: 2}) + assert.Nil(t, err) + assert.Len(t, children, 2) + if assert.NotNil(t, nextPage) { + assert.Equal(t, 2, nextPage.Page) + } +} + +func TestListBitbucketRepos_LastPageHasNoNextPage(t *testing.T) { + client := &fakeApiClient{body: `{"pagelen":2,"page":2,"size":4, + "values":[{"name":"repo-c","full_name":"myworkspace/repo-c"}, + {"name":"repo-d","full_name":"myworkspace/repo-d"}]}`} + children, nextPage, err := listBitbucketRepos(client, "myworkspace", BitbucketRemotePagination{Page: 2, PageLen: 2}) + assert.Nil(t, err) + assert.Len(t, children, 2) + assert.Nil(t, nextPage) +} + +func TestListBitbucketWorkspaces_ReturnsNextPage(t *testing.T) { + client := &fakeApiClient{body: `{"pagelen":1,"page":1,"size":2, + "next":"https://api.bitbucket.org/2.0/user/workspaces?page=2", + "values":[{"workspace":{"slug":"ws-a","name":"Workspace A"}}]}`} + children, nextPage, err := listBitbucketWorkspaces(client, BitbucketRemotePagination{Page: 1, PageLen: 1}) + assert.Nil(t, err) + assert.Len(t, children, 1) + if assert.NotNil(t, nextPage) { + assert.Equal(t, 2, nextPage.Page) + } +} diff --git a/backend/plugins/bitbucket/models/repo.go b/backend/plugins/bitbucket/models/repo.go index 799b549b923..126990a94ab 100644 --- a/backend/plugins/bitbucket/models/repo.go +++ b/backend/plugins/bitbucket/models/repo.go @@ -114,20 +114,19 @@ type WorkspaceResponse struct { Pagelen int `json:"pagelen"` Page int `json:"page"` Size int `json:"size"` + Next string `json:"next"` Values []GroupResponse `json:"values"` } +// GroupResponse maps an entry from GET /2.0/user/workspaces, the supported +// replacement after Bitbucket CHANGE-2770 removed the cross-workspace +// GET /2.0/user/permissions/workspaces and deprecated GET /2.0/workspaces. +// Each value nests the workspace slug/name under a "workspace" object. type GroupResponse struct { - //Type string `json:"type"` - //Permission string `json:"permission"` - //LastAccessed time.Time `json:"last_accessed"` - //AddedOn time.Time `json:"added_on"` Workspace WorkspaceItem `json:"workspace"` } type WorkspaceItem struct { - //Type string `json:"type"` - //Uuid string `json:"uuid"` Slug string `json:"slug" group:"id"` Name string `json:"name" group:"name"` } @@ -144,6 +143,7 @@ type ReposResponse struct { Pagelen int `json:"pagelen"` Page int `json:"page"` Size int `json:"size"` + Next string `json:"next"` Values []BitbucketApiRepo `json:"values"` } diff --git a/backend/plugins/bitbucket/tasks/api_common.go b/backend/plugins/bitbucket/tasks/api_common.go index 352f41cb952..ad21c9b1e65 100644 --- a/backend/plugins/bitbucket/tasks/api_common.go +++ b/backend/plugins/bitbucket/tasks/api_common.go @@ -238,7 +238,10 @@ func ignoreHTTPStatus404(res *http.Response) errors.Error { if res.StatusCode == http.StatusUnauthorized { return errors.Unauthorized.New("authentication failed, please check your AccessToken") } - if res.StatusCode == http.StatusNotFound { + // 404: repo has no issue tracker. 410 Gone: Bitbucket has sunset the issue + // tracker/wiki API, so the endpoint is permanently removed for the repo. + // Both mean "nothing to collect" — skip gracefully instead of retrying. + if res.StatusCode == http.StatusNotFound || res.StatusCode == http.StatusGone { return api.ErrIgnoreAndContinue } return nil diff --git a/backend/plugins/bitbucket/tasks/api_common_test.go b/backend/plugins/bitbucket/tasks/api_common_test.go new file mode 100644 index 00000000000..9d15e52e9c4 --- /dev/null +++ b/backend/plugins/bitbucket/tasks/api_common_test.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "net/http" + "testing" + + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/stretchr/testify/assert" +) + +func TestIgnoreHTTPStatus404(t *testing.T) { + cases := []struct { + name string + statusCode int + wantIgnore bool // expect ErrIgnoreAndContinue (graceful skip, no retry) + wantErr bool // expect a real error + }{ + {"404 no issue tracker -> ignore", http.StatusNotFound, true, false}, + {"410 issue tracker sunset -> ignore", http.StatusGone, true, false}, + {"401 unauthorized -> error", http.StatusUnauthorized, false, true}, + {"200 ok -> continue", http.StatusOK, false, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := ignoreHTTPStatus404(&http.Response{StatusCode: tc.statusCode}) + switch { + case tc.wantIgnore: + assert.Equal(t, api.ErrIgnoreAndContinue, err) + case tc.wantErr: + assert.Error(t, err) + default: + assert.NoError(t, err) + } + }) + } +} diff --git a/backend/plugins/claude_code/models/connection.go b/backend/plugins/claude_code/models/connection.go index 973b710afd6..700b611ae13 100644 --- a/backend/plugins/claude_code/models/connection.go +++ b/backend/plugins/claude_code/models/connection.go @@ -62,6 +62,14 @@ func (conn *ClaudeCodeConn) HasUsableCustomHeaders() bool { return false } +// IsConsoleApiKey returns true when the token is a Claude Console Admin API key. +// Console keys (sk-ant-admin01-...) have full Admin API access and use +// /v1/organizations/usage_report/claude_code instead of the Enterprise +// /v1/organizations/analytics/* endpoints. +func (conn *ClaudeCodeConn) IsConsoleApiKey() bool { + return strings.HasPrefix(strings.TrimSpace(conn.Token), "sk-ant-admin01-") +} + func (conn *ClaudeCodeConn) HasIncompleteCustomHeaders() bool { if conn == nil { return false diff --git a/backend/plugins/claude_code/tasks/activity_summary_collector.go b/backend/plugins/claude_code/tasks/activity_summary_collector.go index 110c44e54b5..79728f19090 100644 --- a/backend/plugins/claude_code/tasks/activity_summary_collector.go +++ b/backend/plugins/claude_code/tasks/activity_summary_collector.go @@ -44,6 +44,11 @@ func CollectActivitySummary(taskCtx plugin.SubTaskContext) errors.Error { return nil } + if connection.IsConsoleApiKey() { + taskCtx.GetLogger().Info("Console API key detected, skipping activity summary collection (no equivalent endpoint)") + return nil + } + apiClient, err := CreateApiClient(taskCtx.TaskContext(), connection) if err != nil { return err diff --git a/backend/plugins/claude_code/tasks/connector_usage_collector.go b/backend/plugins/claude_code/tasks/connector_usage_collector.go index 076cf00e178..2b885b1ef57 100644 --- a/backend/plugins/claude_code/tasks/connector_usage_collector.go +++ b/backend/plugins/claude_code/tasks/connector_usage_collector.go @@ -44,6 +44,11 @@ func CollectConnectorUsage(taskCtx plugin.SubTaskContext) errors.Error { return nil } + if connection.IsConsoleApiKey() { + taskCtx.GetLogger().Info("Console API key detected, skipping connector usage collection (no equivalent endpoint)") + return nil + } + apiClient, err := CreateApiClient(taskCtx.TaskContext(), connection) if err != nil { return err diff --git a/backend/plugins/claude_code/tasks/user_activity_collector.go b/backend/plugins/claude_code/tasks/user_activity_collector.go index 16a76472126..7239390be52 100644 --- a/backend/plugins/claude_code/tasks/user_activity_collector.go +++ b/backend/plugins/claude_code/tasks/user_activity_collector.go @@ -30,7 +30,10 @@ import ( helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" ) -// CollectUserActivity collects per-user daily engagement metrics from /v1/organizations/analytics/users. +// CollectUserActivity collects per-user daily engagement metrics. +// For Claude Enterprise organizations it calls /v1/organizations/analytics/users. +// For Claude Console organizations (sk-ant-admin01-... keys) it calls +// /v1/organizations/usage_report/claude_code. func CollectUserActivity(taskCtx plugin.SubTaskContext) errors.Error { data, ok := taskCtx.TaskContext().GetData().(*ClaudeCodeTaskData) if !ok { @@ -49,6 +52,14 @@ func CollectUserActivity(taskCtx plugin.SubTaskContext) errors.Error { return err } + isConsole := connection.IsConsoleApiKey() + endpoint := "analytics/users" + urlTemplate := "v1/organizations/analytics/users" + if isConsole { + endpoint = "usage_report/claude_code" + urlTemplate = "v1/organizations/usage_report/claude_code" + } + rawArgs := helper.RawDataSubTaskArgs{ Ctx: taskCtx, Table: rawUserActivityTable, @@ -56,7 +67,7 @@ func CollectUserActivity(taskCtx plugin.SubTaskContext) errors.Error { ConnectionId: data.Options.ConnectionId, ScopeId: data.Options.ScopeId, Organization: connection.Organization, - Endpoint: "analytics/users", + Endpoint: endpoint, }, } @@ -73,12 +84,16 @@ func CollectUserActivity(taskCtx plugin.SubTaskContext) errors.Error { Input: dayIter, PageSize: 1, Incremental: true, - UrlTemplate: "v1/organizations/analytics/users", + UrlTemplate: urlTemplate, GetNextPageCustomData: getNextClaudeCodePageCursor, Query: func(reqData *helper.RequestData) (url.Values, errors.Error) { input := reqData.Input.(*claudeCodeDayInput) query := url.Values{} - query.Set("date", input.Day) + if isConsole { + query.Set("starting_at", input.Day) + } else { + query.Set("date", input.Day) + } query.Set("limit", fmt.Sprintf("%d", claudeCodeApiPageLimit)) if cursor, ok := reqData.CustomData.(string); ok && strings.TrimSpace(cursor) != "" { query.Set("page", cursor) diff --git a/backend/plugins/claude_code/tasks/user_activity_extractor.go b/backend/plugins/claude_code/tasks/user_activity_extractor.go index 040b5a8d496..4050a7c2b1f 100644 --- a/backend/plugins/claude_code/tasks/user_activity_extractor.go +++ b/backend/plugins/claude_code/tasks/user_activity_extractor.go @@ -28,6 +28,8 @@ import ( "github.com/apache/incubator-devlake/plugins/claude_code/models" ) +// ── Enterprise API response types (/v1/organizations/analytics/users) ────────── + // userActivityRecord is the JSON shape returned by /v1/organizations/analytics/users. type userActivityRecord struct { User userActivityUser `json:"user"` @@ -82,6 +84,48 @@ type userActivityCCTools struct { NotebookEditTool userActivityToolAction `json:"notebook_edit_tool"` } +// ── Console API response types (/v1/organizations/usage_report/claude_code) ─── + +// consoleUserActivityRecord is the JSON shape returned by /v1/organizations/usage_report/claude_code. +type consoleUserActivityRecord struct { + Date string `json:"date"` + Actor consoleActor `json:"actor"` + CoreMetrics consoleCoreMetrics `json:"core_metrics"` + ToolActions consoleToolActions `json:"tool_actions"` +} + +type consoleActor struct { + Type string `json:"type"` + EmailAddress string `json:"email_address"` + ApiKeyName string `json:"api_key_name"` +} + +type consoleCoreMetrics struct { + NumSessions int `json:"num_sessions"` + LinesOfCode consoleLinesOfCode `json:"lines_of_code"` + CommitsByClaudeCode int `json:"commits_by_claude_code"` + PullRequestsByClaudeCode int `json:"pull_requests_by_claude_code"` +} + +type consoleLinesOfCode struct { + Added int `json:"added"` + Removed int `json:"removed"` +} + +type consoleToolAction struct { + Accepted int `json:"accepted"` + Rejected int `json:"rejected"` +} + +type consoleToolActions struct { + EditTool consoleToolAction `json:"edit_tool"` + MultiEditTool consoleToolAction `json:"multi_edit_tool"` + WriteTool consoleToolAction `json:"write_tool"` + NotebookEditTool consoleToolAction `json:"notebook_edit_tool"` +} + +// ── Extractor ───────────────────────────────────────────────────────────────── + // ExtractUserActivity parses raw user activity records into tool-layer tables. func ExtractUserActivity(taskCtx plugin.SubTaskContext) errors.Error { data, ok := taskCtx.TaskContext().GetData().(*ClaudeCodeTaskData) @@ -96,6 +140,12 @@ func ExtractUserActivity(taskCtx plugin.SubTaskContext) errors.Error { return nil } + isConsole := connection.IsConsoleApiKey() + endpoint := "analytics/users" + if isConsole { + endpoint = "usage_report/claude_code" + } + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ Ctx: taskCtx, @@ -104,63 +154,18 @@ func ExtractUserActivity(taskCtx plugin.SubTaskContext) errors.Error { ConnectionId: data.Options.ConnectionId, ScopeId: data.Options.ScopeId, Organization: connection.Organization, - Endpoint: "analytics/users", + Endpoint: endpoint, }, }, Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { - var record userActivityRecord - if err := errors.Convert(json.Unmarshal(row.Data, &record)); err != nil { - return nil, err - } - date, parseErr := parseAnalyticsDate(row.Input) if parseErr != nil { return nil, parseErr } - - userId := strings.TrimSpace(record.User.Id) - if userId == "" { - userId = strings.TrimSpace(record.User.EmailAddress) - } - if userId == "" { - return nil, nil - } - - activity := &models.ClaudeCodeUserActivity{ - ConnectionId: data.Options.ConnectionId, - ScopeId: data.Options.ScopeId, - Date: date, - UserId: userId, - UserEmail: strings.TrimSpace(record.User.EmailAddress), - - ChatConversationCount: record.ChatMetrics.DistinctConversationCount, - ChatMessageCount: record.ChatMetrics.MessageCount, - ChatProjectsCreatedCount: record.ChatMetrics.DistinctProjectsCreatedCount, - ChatProjectsUsedCount: record.ChatMetrics.DistinctProjectsUsedCount, - ChatFilesUploadedCount: record.ChatMetrics.DistinctFilesUploadedCount, - ChatArtifactsCreatedCount: record.ChatMetrics.DistinctArtifactsCreatedCount, - ChatThinkingMessageCount: record.ChatMetrics.ThinkingMessageCount, - ChatSkillsUsedCount: record.ChatMetrics.DistinctSkillsUsedCount, - ChatConnectorsUsedCount: record.ChatMetrics.ConnectorsUsedCount, - - CCCommitCount: record.ClaudeCodeMetrics.CoreMetrics.CommitCount, - CCPullRequestCount: record.ClaudeCodeMetrics.CoreMetrics.PullRequestCount, - CCLinesAdded: record.ClaudeCodeMetrics.CoreMetrics.LinesOfCode.AddedCount, - CCLinesRemoved: record.ClaudeCodeMetrics.CoreMetrics.LinesOfCode.RemovedCount, - CCSessionCount: record.ClaudeCodeMetrics.CoreMetrics.DistinctSessionCount, - - EditToolAccepted: record.ClaudeCodeMetrics.ToolActions.EditTool.AcceptedCount, - EditToolRejected: record.ClaudeCodeMetrics.ToolActions.EditTool.RejectedCount, - MultiEditToolAccepted: record.ClaudeCodeMetrics.ToolActions.MultiEditTool.AcceptedCount, - MultiEditToolRejected: record.ClaudeCodeMetrics.ToolActions.MultiEditTool.RejectedCount, - WriteToolAccepted: record.ClaudeCodeMetrics.ToolActions.WriteTool.AcceptedCount, - WriteToolRejected: record.ClaudeCodeMetrics.ToolActions.WriteTool.RejectedCount, - NotebookEditToolAccepted: record.ClaudeCodeMetrics.ToolActions.NotebookEditTool.AcceptedCount, - NotebookEditToolRejected: record.ClaudeCodeMetrics.ToolActions.NotebookEditTool.RejectedCount, - - WebSearchCount: record.WebSearchCount, + if isConsole { + return extractConsoleUserActivity(data, date, row.Data) } - return []interface{}{activity}, nil + return extractEnterpriseUserActivity(data, date, row.Data) }, }) if err != nil { @@ -169,10 +174,99 @@ func ExtractUserActivity(taskCtx plugin.SubTaskContext) errors.Error { return extractor.Execute() } +func extractEnterpriseUserActivity(data *ClaudeCodeTaskData, date time.Time, raw []byte) ([]interface{}, errors.Error) { + var record userActivityRecord + if err := errors.Convert(json.Unmarshal(raw, &record)); err != nil { + return nil, err + } + + userId := strings.TrimSpace(record.User.Id) + if userId == "" { + userId = strings.TrimSpace(record.User.EmailAddress) + } + if userId == "" { + return nil, nil + } + + activity := &models.ClaudeCodeUserActivity{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Date: date, + UserId: userId, + UserEmail: strings.TrimSpace(record.User.EmailAddress), + + ChatConversationCount: record.ChatMetrics.DistinctConversationCount, + ChatMessageCount: record.ChatMetrics.MessageCount, + ChatProjectsCreatedCount: record.ChatMetrics.DistinctProjectsCreatedCount, + ChatProjectsUsedCount: record.ChatMetrics.DistinctProjectsUsedCount, + ChatFilesUploadedCount: record.ChatMetrics.DistinctFilesUploadedCount, + ChatArtifactsCreatedCount: record.ChatMetrics.DistinctArtifactsCreatedCount, + ChatThinkingMessageCount: record.ChatMetrics.ThinkingMessageCount, + ChatSkillsUsedCount: record.ChatMetrics.DistinctSkillsUsedCount, + ChatConnectorsUsedCount: record.ChatMetrics.ConnectorsUsedCount, + + CCCommitCount: record.ClaudeCodeMetrics.CoreMetrics.CommitCount, + CCPullRequestCount: record.ClaudeCodeMetrics.CoreMetrics.PullRequestCount, + CCLinesAdded: record.ClaudeCodeMetrics.CoreMetrics.LinesOfCode.AddedCount, + CCLinesRemoved: record.ClaudeCodeMetrics.CoreMetrics.LinesOfCode.RemovedCount, + CCSessionCount: record.ClaudeCodeMetrics.CoreMetrics.DistinctSessionCount, + + EditToolAccepted: record.ClaudeCodeMetrics.ToolActions.EditTool.AcceptedCount, + EditToolRejected: record.ClaudeCodeMetrics.ToolActions.EditTool.RejectedCount, + MultiEditToolAccepted: record.ClaudeCodeMetrics.ToolActions.MultiEditTool.AcceptedCount, + MultiEditToolRejected: record.ClaudeCodeMetrics.ToolActions.MultiEditTool.RejectedCount, + WriteToolAccepted: record.ClaudeCodeMetrics.ToolActions.WriteTool.AcceptedCount, + WriteToolRejected: record.ClaudeCodeMetrics.ToolActions.WriteTool.RejectedCount, + NotebookEditToolAccepted: record.ClaudeCodeMetrics.ToolActions.NotebookEditTool.AcceptedCount, + NotebookEditToolRejected: record.ClaudeCodeMetrics.ToolActions.NotebookEditTool.RejectedCount, + + WebSearchCount: record.WebSearchCount, + } + return []interface{}{activity}, nil +} + +func extractConsoleUserActivity(data *ClaudeCodeTaskData, date time.Time, raw []byte) ([]interface{}, errors.Error) { + var record consoleUserActivityRecord + if err := errors.Convert(json.Unmarshal(raw, &record)); err != nil { + return nil, err + } + + userId := strings.TrimSpace(record.Actor.EmailAddress) + if userId == "" { + userId = strings.TrimSpace(record.Actor.ApiKeyName) + } + if userId == "" { + return nil, nil + } + + activity := &models.ClaudeCodeUserActivity{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Date: date, + UserId: userId, + UserEmail: strings.TrimSpace(record.Actor.EmailAddress), + + CCSessionCount: record.CoreMetrics.NumSessions, + CCCommitCount: record.CoreMetrics.CommitsByClaudeCode, + CCPullRequestCount: record.CoreMetrics.PullRequestsByClaudeCode, + CCLinesAdded: record.CoreMetrics.LinesOfCode.Added, + CCLinesRemoved: record.CoreMetrics.LinesOfCode.Removed, + + EditToolAccepted: record.ToolActions.EditTool.Accepted, + EditToolRejected: record.ToolActions.EditTool.Rejected, + MultiEditToolAccepted: record.ToolActions.MultiEditTool.Accepted, + MultiEditToolRejected: record.ToolActions.MultiEditTool.Rejected, + WriteToolAccepted: record.ToolActions.WriteTool.Accepted, + WriteToolRejected: record.ToolActions.WriteTool.Rejected, + NotebookEditToolAccepted: record.ToolActions.NotebookEditTool.Accepted, + NotebookEditToolRejected: record.ToolActions.NotebookEditTool.Rejected, + } + return []interface{}{activity}, nil +} + // parseAnalyticsDate extracts the date from the raw row input JSON. // The input is the claudeCodeDayInput or claudeCodeDateRangeInput encoded as JSON. func parseAnalyticsDate(rawInput json.RawMessage) (time.Time, errors.Error) { - // Try day input first. var dayInput claudeCodeDayInput if err := json.Unmarshal(rawInput, &dayInput); err == nil && dayInput.Day != "" { t, parseErr := time.Parse("2006-01-02", strings.TrimSpace(dayInput.Day)) @@ -180,7 +274,6 @@ func parseAnalyticsDate(rawInput json.RawMessage) (time.Time, errors.Error) { return utcDate(t), nil } } - // Fall back to date range input (summaries). var rangeInput claudeCodeDateRangeInput if err := json.Unmarshal(rawInput, &rangeInput); err == nil && rangeInput.StartDate != "" { t, parseErr := time.Parse("2006-01-02", strings.TrimSpace(rangeInput.StartDate)) diff --git a/backend/plugins/clickup/README.md b/backend/plugins/clickup/README.md new file mode 100644 index 00000000000..1d4122a5bbb --- /dev/null +++ b/backend/plugins/clickup/README.md @@ -0,0 +1,92 @@ + + +# ClickUp + +The ClickUp plugin collects issues, boards, and sprints from ClickUp so they +feed DevLake's issue-tracking and DORA/velocity metrics, modeled on the Jira +and Linear connectors. + +## Authentication + +ClickUp authenticates with a **personal API token** (ClickUp → Settings → Apps +→ API Token, starts with `pk_`). The token is sent verbatim in the +`Authorization` header. OAuth is not yet supported. + +Create a connection with: + +- **Endpoint** — `https://api.clickup.com/api/v2/` +- **Token** — your `pk_...` personal token + +## Data scope: the folder is the board + +Unlike a raw list, the **scope you select is a ClickUp folder** (e.g. a team's +`Dev Team` / `Sprint Folder`). This mirrors a Jira board: selecting the folder +collects every list inside it on each sync, so rolling and archived sprint +lists never need to be re-scoped. + +Domain mapping: + +| ClickUp | DevLake domain | +| --- | --- | +| Folder | `board` | +| Sprint list (name-matched) | `sprint` + `board_sprint` | +| Task | `issue` + `board_issue` | +| Task in a sprint list | `sprint_issue` | +| Folder member | `account` | + +## Sprints + +A list is treated as a sprint when its name matches the **sprint name pattern** +(default `(?i)sprint\s*\d+`, e.g. `v4.3.0 Sprint 40 (7/6/26 - 7/19/26)`). The +start/end dates are parsed from the parenthesised date span in the name; +`M/D/YY` vs `D/M/YY` ordering is disambiguated automatically (a component > 12 +must be the day). Lists that don't match are collected as plain board issues. +Archived sprint lists are collected too, so historical velocity is retained. + +## Story points + +Story points default to ClickUp's native sprint **`points`** field. To read +them from a custom field instead (e.g. a Fibonacci "LOE" field), set +**Story point field** in the scope config to the custom-field name. + +## Incidents (DORA Change Failure Rate / MTTR) + +ClickUp tasks have no universal "type" field, so incidents are modeled by +**scope**: add the folder that holds incidents (e.g. a "Security Incidents" +folder) as its own board and set the scope config's **Force issue type** to +`INCIDENT`. Every issue on that board is then classified as an incident. + +## Scope configuration (transformation) + +Per board, the scope config lets you override: + +- **Sprint name pattern** — which lists are sprints. +- **Story point field** — native `points` (blank) or a custom field name. +- **Force issue type** — flag the whole board as `REQUIREMENT` / `BUG` / + `INCIDENT` (leave blank to detect per task). +- **Issue type patterns** — RegExes matched against a task's type; precedence + is `INCIDENT` > `BUG` > `REQUIREMENT`. +- **Status mapping** — ClickUp statuses are auto-mapped by their `type` + (`open`/`unstarted` → `TODO`, `custom` → `IN_PROGRESS`, `done`/`closed` → + `DONE`); list raw status names to override where a team's workflow differs. + +## Metrics + +A branded **ClickUp** Grafana dashboard (`grafana/dashboards/{mysql,postgresql}/ClickUp.json`) +renders issue throughput and lead-time metrics; DORA and sprint/velocity +metrics are computed from the source-agnostic domain layer. diff --git a/backend/plugins/clickup/api/blueprint_v200.go b/backend/plugins/clickup/api/blueprint_v200.go new file mode 100644 index 00000000000..98f2439d696 --- /dev/null +++ b/backend/plugins/clickup/api/blueprint_v200.go @@ -0,0 +1,99 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/clickup/models" + "github.com/apache/incubator-devlake/plugins/clickup/tasks" +) + +func MakePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + connectionId uint64, + bpScopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + connection, err := dsHelper.ConnSrv.FindByPk(connectionId) + if err != nil { + return nil, nil, err + } + scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes) + if err != nil { + return nil, nil, err + } + plan, err := makePipelinePlanV200(subtaskMetas, scopeDetails, connection) + if err != nil { + return nil, nil, err + } + scopes, err := makeScopesV200(scopeDetails, connection) + return plan, scopes, err +} + +func makePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + scopeDetails []*srvhelper.ScopeDetail[models.ClickUpFolder, models.ClickUpScopeConfig], + connection *models.ClickUpConnection, +) (coreModels.PipelinePlan, errors.Error) { + plan := make(coreModels.PipelinePlan, len(scopeDetails)) + for i, scopeDetail := range scopeDetails { + stage := plan[i] + if stage == nil { + stage = coreModels.PipelineStage{} + } + scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig + task, err := helper.MakePipelinePlanTask( + "clickup", + subtaskMetas, + scopeConfig.Entities, + tasks.ClickUpOptions{ + ConnectionId: connection.ID, + FolderId: scope.FolderId, + ScopeConfigId: scope.ScopeConfigId, + }, + ) + if err != nil { + return nil, err + } + stage = append(stage, task) + plan[i] = stage + } + return plan, nil +} + +func makeScopesV200( + scopeDetails []*srvhelper.ScopeDetail[models.ClickUpFolder, models.ClickUpScopeConfig], + connection *models.ClickUpConnection, +) ([]plugin.Scope, errors.Error) { + scopes := make([]plugin.Scope, 0, len(scopeDetails)) + idgen := didgen.NewDomainIdGenerator(&models.ClickUpFolder{}) + for _, scopeDetail := range scopeDetails { + scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig + id := idgen.Generate(connection.ID, scope.FolderId) + if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_TICKET) { + scopes = append(scopes, ticket.NewBoard(id, scope.Name)) + } + } + return scopes, nil +} diff --git a/backend/plugins/clickup/api/connection_api.go b/backend/plugins/clickup/api/connection_api.go new file mode 100644 index 00000000000..052b1e5f68a --- /dev/null +++ b/backend/plugins/clickup/api/connection_api.go @@ -0,0 +1,175 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" + "github.com/apache/incubator-devlake/plugins/clickup/tasks" + "github.com/apache/incubator-devlake/server/api/shared" +) + +type ClickUpTestConnResponse struct { + shared.ApiBody + Connection *models.ClickUpConn +} + +func testConnection(ctx context.Context, connection models.ClickUpConn) (*ClickUpTestConnResponse, errors.Error) { + if vld != nil { + if err := vld.Struct(connection); err != nil { + return nil, errors.Default.Wrap(err, "error validating target") + } + } + if connection.Endpoint == "" { + connection.Endpoint = tasks.DefaultEndpoint + } + apiClient, err := helper.NewApiClientFromConnection(ctx, basicRes, &connection) + if err != nil { + return nil, err + } + // GET /team lists the authenticated user's workspaces; it verifies the token. + res, err := apiClient.Get("team", nil, nil) + if err != nil { + return nil, errors.BadInput.Wrap(err, "verify token failed") + } + if res.StatusCode == http.StatusUnauthorized || res.StatusCode == http.StatusForbidden { + return nil, errors.HttpStatus(http.StatusBadRequest).New("authentication failed, please check your API token") + } + if res.StatusCode != http.StatusOK { + return nil, errors.HttpStatus(res.StatusCode).New("unexpected status code while testing connection") + } + connection = connection.Sanitize() + body := ClickUpTestConnResponse{} + body.Success = true + body.Message = "success" + body.Connection = &connection + return &body, nil +} + +// TestConnection test clickup connection +// @Summary test clickup connection +// @Description Test clickup Connection +// @Tags plugins/clickup +// @Param body body models.ClickUpConn true "json body" +// @Success 200 {object} ClickUpTestConnResponse "Success" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/clickup/test [POST] +func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + var connection models.ClickUpConn + if err := helper.Decode(input.Body, &connection, vld); err != nil { + return nil, err + } + result, err := testConnection(context.TODO(), connection) + if err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} + +// TestExistingConnection test clickup connection by ID +// @Summary test clickup connection +// @Description Test clickup Connection +// @Tags plugins/clickup +// @Param connectionId path int true "connection ID" +// @Success 200 {object} ClickUpTestConnResponse "Success" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/clickup/connections/{connectionId}/test [POST] +func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection, err := dsHelper.ConnApi.GetMergedConnection(input) + if err != nil { + return nil, errors.BadInput.Wrap(err, "find connection from db") + } + if err := helper.DecodeMapStruct(input.Body, connection, false); err != nil { + return nil, err + } + result, testErr := testConnection(context.TODO(), connection.ClickUpConn) + if testErr != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, testErr) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} + +// PostConnections create clickup connection +// @Summary create clickup connection +// @Description Create clickup connection +// @Tags plugins/clickup +// @Param body body models.ClickUpConnection true "json body" +// @Success 200 {object} models.ClickUpConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/clickup/connections [POST] +func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Post(input) +} + +// PatchConnection patch clickup connection +// @Summary patch clickup connection +// @Description Patch clickup connection +// @Tags plugins/clickup +// @Param body body models.ClickUpConnection true "json body" +// @Success 200 {object} models.ClickUpConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/clickup/connections/{connectionId} [PATCH] +func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Patch(input) +} + +// DeleteConnection delete a clickup connection +// @Summary delete a clickup connection +// @Description Delete a clickup connection +// @Tags plugins/clickup +// @Success 200 {object} models.ClickUpConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 409 {object} services.BlueprintProjectPairs "References exist to this connection" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/clickup/connections/{connectionId} [DELETE] +func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Delete(input) +} + +// ListConnections get all clickup connections +// @Summary get all clickup connections +// @Description Get all clickup connections +// @Tags plugins/clickup +// @Success 200 {object} []models.ClickUpConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/clickup/connections [GET] +func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetAll(input) +} + +// GetConnection get clickup connection detail +// @Summary get clickup connection detail +// @Description Get clickup connection detail +// @Tags plugins/clickup +// @Success 200 {object} models.ClickUpConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/clickup/connections/{connectionId} [GET] +func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetDetail(input) +} diff --git a/backend/plugins/clickup/api/init.go b/backend/plugins/clickup/api/init.go new file mode 100644 index 00000000000..bc185fbd5b0 --- /dev/null +++ b/backend/plugins/clickup/api/init.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" + "github.com/go-playground/validator/v10" +) + +var vld *validator.Validate +var basicRes context.BasicRes +var dsHelper *api.DsHelper[models.ClickUpConnection, models.ClickUpFolder, models.ClickUpScopeConfig] +var raProxy *api.DsRemoteApiProxyHelper[models.ClickUpConnection] +var raScopeList *api.DsRemoteApiScopeListHelper[models.ClickUpConnection, models.ClickUpFolder, ClickUpRemotePagination] + +func Init(br context.BasicRes, p plugin.PluginMeta) { + basicRes = br + vld = validator.New() + dsHelper = api.NewDataSourceHelper[ + models.ClickUpConnection, models.ClickUpFolder, models.ClickUpScopeConfig, + ]( + br, + p.Name(), + []string{"name"}, + func(c models.ClickUpConnection) models.ClickUpConnection { + return c.Sanitize() + }, + nil, + nil, + ) + raProxy = api.NewDsRemoteApiProxyHelper[models.ClickUpConnection](dsHelper.ConnApi.ModelApiHelper) + raScopeList = api.NewDsRemoteApiScopeListHelper[models.ClickUpConnection, models.ClickUpFolder, ClickUpRemotePagination](raProxy, listClickUpRemoteScopes) +} diff --git a/backend/plugins/clickup/api/remote_api.go b/backend/plugins/clickup/api/remote_api.go new file mode 100644 index 00000000000..93e9589c93b --- /dev/null +++ b/backend/plugins/clickup/api/remote_api.go @@ -0,0 +1,170 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "fmt" + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + dsmodels "github.com/apache/incubator-devlake/helpers/pluginhelper/api/models" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +// ClickUpRemotePagination is a placeholder: the ClickUp v2 hierarchy endpoints +// used here (team/space/folder) are not paginated, so there is never a next +// page. It exists to satisfy the DsRemoteApiScopeListHelper generic. +type ClickUpRemotePagination struct{} + +// groupId prefixes encode which level of the ClickUp hierarchy a group entry +// points at, so a single list function can walk Team -> Space -> Folder. The +// selectable scope is the Folder (= board); spaces and teams are navigation +// groups only. +const ( + groupTeamPrefix = "team:" + groupSpacePrefix = "space:" +) + +type clickUpNamedEntities struct { + Teams []clickUpNamedEntity `json:"teams"` + Spaces []clickUpNamedEntity `json:"spaces"` + Folders []clickUpNamedEntity `json:"folders"` +} + +type clickUpNamedEntity struct { + Id string `json:"id"` + Name string `json:"name"` +} + +// listClickUpRemoteScopes walks the ClickUp hierarchy one level at a time. The +// config UI drives it via `groupId`: +// +// "" -> workspaces (Team) as groups +// team:{id} -> spaces as groups +// space:{id} -> folders as selectable scopes (boards) +func listClickUpRemoteScopes( + _ *models.ClickUpConnection, + apiClient plugin.ApiClient, + groupId string, + _ ClickUpRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.ClickUpFolder], + nextPage *ClickUpRemotePagination, + err errors.Error, +) { + switch { + case groupId == "": + return listTeamsAsGroups(apiClient) + case strings.HasPrefix(groupId, groupTeamPrefix): + return listSpacesAsGroups(apiClient, strings.TrimPrefix(groupId, groupTeamPrefix), groupId) + case strings.HasPrefix(groupId, groupSpacePrefix): + return listFoldersAsScopes(apiClient, strings.TrimPrefix(groupId, groupSpacePrefix), groupId) + default: + return nil, nil, errors.BadInput.New(fmt.Sprintf("unrecognized groupId %q", groupId)) + } +} + +func getEntities(apiClient plugin.ApiClient, path string) (*clickUpNamedEntities, errors.Error) { + res, err := apiClient.Get(path, nil, nil) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to query ClickUp "+path) + } + var body clickUpNamedEntities + if err := api.UnmarshalResponse(res, &body); err != nil { + return nil, errors.Default.Wrap(err, "failed to unmarshal ClickUp "+path+" response") + } + return &body, nil +} + +// groupEntry builds a navigation-group row. parentId is the id of the group +// this row is nested under (nil for the top level); the miller-column UI uses +// it to render children in the next column instead of inline. +func groupEntry(id, name string, parentId *string) dsmodels.DsRemoteApiScopeListEntry[models.ClickUpFolder] { + return dsmodels.DsRemoteApiScopeListEntry[models.ClickUpFolder]{ + Type: api.RAS_ENTRY_TYPE_GROUP, + ParentId: parentId, + Id: id, + Name: name, + FullName: name, + } +} + +func listTeamsAsGroups(apiClient plugin.ApiClient) ([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpFolder], *ClickUpRemotePagination, errors.Error) { + body, err := getEntities(apiClient, "team") + if err != nil { + return nil, nil, err + } + children := make([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpFolder], 0, len(body.Teams)) + for _, team := range body.Teams { + children = append(children, groupEntry(groupTeamPrefix+team.Id, team.Name, nil)) + } + return children, nil, nil +} + +func listSpacesAsGroups(apiClient plugin.ApiClient, teamId, parentId string) ([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpFolder], *ClickUpRemotePagination, errors.Error) { + body, err := getEntities(apiClient, fmt.Sprintf("team/%s/space", teamId)) + if err != nil { + return nil, nil, err + } + children := make([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpFolder], 0, len(body.Spaces)) + for _, space := range body.Spaces { + children = append(children, groupEntry(groupSpacePrefix+space.Id, space.Name, &parentId)) + } + return children, nil, nil +} + +// listFoldersAsScopes returns a space's folders as selectable (leaf) scope +// entries — the folder is the board a user picks. parentId nests them under the +// space column. +func listFoldersAsScopes(apiClient plugin.ApiClient, spaceId, parentId string) ([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpFolder], *ClickUpRemotePagination, errors.Error) { + body, err := getEntities(apiClient, fmt.Sprintf("space/%s/folder", spaceId)) + if err != nil { + return nil, nil, err + } + entries := make([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpFolder], 0, len(body.Folders)) + for _, folder := range body.Folders { + folder := folder + entries = append(entries, dsmodels.DsRemoteApiScopeListEntry[models.ClickUpFolder]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + ParentId: &parentId, + Id: folder.Id, + Name: folder.Name, + FullName: folder.Name, + Data: &models.ClickUpFolder{ + FolderId: folder.Id, + Name: folder.Name, + SpaceId: spaceId, + SpaceName: "", + }, + }) + } + return entries, nil, nil +} + +// RemoteScopes lists the ClickUp folders available on the connection so the +// config UI can enumerate selectable scopes. +func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raScopeList.Get(input) +} + +// Proxy forwards arbitrary requests to the ClickUp API through the connection. +func Proxy(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raProxy.Proxy(input) +} diff --git a/backend/plugins/clickup/api/scope_api.go b/backend/plugins/clickup/api/scope_api.go new file mode 100644 index 00000000000..5bfcb9d5aef --- /dev/null +++ b/backend/plugins/clickup/api/scope_api.go @@ -0,0 +1,105 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +type PutScopesReqBody api.PutScopesReqBody[models.ClickUpFolder] +type ScopeDetail api.ScopeDetail[models.ClickUpFolder, models.ClickUpScopeConfig] + +// PutScopes create or update clickup lists +// @Summary create or update clickup lists +// @Description Create or update clickup lists +// @Tags plugins/clickup +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param scope body PutScopesReqBody true "json" +// @Success 200 {object} []models.ClickUpFolder +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/clickup/connections/{connectionId}/scopes [PUT] +func PutScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.PutMultiple(input) +} + +// PatchScope patch to clickup list +// @Summary patch to clickup list +// @Description patch to clickup list +// @Tags plugins/clickup +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param scopeId path string false "list ID" +// @Param scope body models.ClickUpFolder true "json" +// @Success 200 {object} models.ClickUpFolder +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/clickup/connections/{connectionId}/scopes/{scopeId} [PATCH] +func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Patch(input) +} + +// GetScopeList get clickup lists +// @Summary get clickup lists +// @Description get clickup lists +// @Tags plugins/clickup +// @Param connectionId path int false "connection ID" +// @Param searchTerm query string false "search term for scope name" +// @Param pageSize query int false "page size, default 50" +// @Param page query int false "page size, default 1" +// @Success 200 {object} []ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/clickup/connections/{connectionId}/scopes/ [GET] +func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetPage(input) +} + +// GetScope get one clickup list +// @Summary get one clickup list +// @Description get one clickup list +// @Tags plugins/clickup +// @Param connectionId path int false "connection ID" +// @Param scopeId path string false "list ID" +// @Success 200 {object} ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/clickup/connections/{connectionId}/scopes/{scopeId} [GET] +func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeDetail(input) +} + +// DeleteScope delete plugin data associated with the scope and optionally the scope itself +// @Summary delete plugin data associated with the scope and optionally the scope itself +// @Description delete data associated with plugin scope +// @Tags plugins/clickup +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Param delete_data_only query bool false "Only delete the scope data, not the scope itself" +// @Success 200 +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 409 {object} api.ScopeRefDoc "References exist to this scope" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/clickup/connections/{connectionId}/scopes/{scopeId} [DELETE] +func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Delete(input) +} diff --git a/backend/plugins/clickup/api/scope_config_api.go b/backend/plugins/clickup/api/scope_config_api.go new file mode 100644 index 00000000000..fab37352f5c --- /dev/null +++ b/backend/plugins/clickup/api/scope_config_api.go @@ -0,0 +1,106 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +// PostScopeConfig create scope config for ClickUp +// @Summary create scope config for ClickUp +// @Description create scope config for ClickUp +// @Tags plugins/clickup +// @Accept application/json +// @Param scopeConfig body models.ClickUpScopeConfig true "scope config" +// @Success 200 {object} models.ClickUpScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/clickup/connections/{connectionId}/scope-configs [POST] +func PostScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Post(input) +} + +// PatchScopeConfig update scope config for ClickUp +// @Summary update scope config for ClickUp +// @Description update scope config for ClickUp +// @Tags plugins/clickup +// @Accept application/json +// @Param scopeConfigId path int true "scopeConfigId" +// @Param scopeConfig body models.ClickUpScopeConfig true "scope config" +// @Success 200 {object} models.ClickUpScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/clickup/connections/{connectionId}/scope-configs/{scopeConfigId} [PATCH] +func PatchScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Patch(input) +} + +// GetScopeConfig return one scope config +// @Summary return one scope config +// @Description return one scope config +// @Tags plugins/clickup +// @Param scopeConfigId path int true "scopeConfigId" +// @Success 200 {object} models.ClickUpScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/clickup/connections/{connectionId}/scope-configs/{scopeConfigId} [GET] +func GetScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetDetail(input) +} + +// GetScopeConfigList return all scope configs +// @Summary return all scope configs +// @Description return all scope configs +// @Tags plugins/clickup +// @Param pageSize query int false "page size, default 50" +// @Param page query int false "page size, default 1" +// @Success 200 {object} []models.ClickUpScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/clickup/connections/{connectionId}/scope-configs [GET] +func GetScopeConfigList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetAll(input) +} + +// DeleteScopeConfig delete a scope config +// @Summary delete a scope config +// @Description delete a scope config +// @Tags plugins/clickup +// @Param scopeConfigId path int true "scopeConfigId" +// @Param connectionId path int true "connectionId" +// @Success 200 +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/clickup/connections/{connectionId}/scope-configs/{scopeConfigId} [DELETE] +func DeleteScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Delete(input) +} + +// GetProjectsByScopeConfig return projects details related by scope config +// @Summary return all related projects +// @Description return all related projects +// @Tags plugins/clickup +// @Param scopeConfigId path int true "scopeConfigId" +// @Success 200 {object} models.ProjectScopeOutput +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/clickup/scope-config/{scopeConfigId}/projects [GET] +func GetProjectsByScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetProjectsByScopeConfig(input) +} diff --git a/backend/plugins/clickup/clickup.go b/backend/plugins/clickup/clickup.go new file mode 100644 index 00000000000..a7806a1c7d7 --- /dev/null +++ b/backend/plugins/clickup/clickup.go @@ -0,0 +1,43 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main // must be main for plugin entry point + +import ( + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/clickup/impl" + "github.com/spf13/cobra" +) + +var PluginEntry impl.ClickUp //nolint + +// standalone mode for debugging +func main() { + cmd := &cobra.Command{Use: "clickup"} + connectionId := cmd.Flags().Uint64P("connection", "c", 0, "clickup connection id") + listId := cmd.Flags().StringP("list", "l", "", "clickup list id") + timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z") + _ = cmd.MarkFlagRequired("connection") + _ = cmd.MarkFlagRequired("list") + cmd.Run = func(c *cobra.Command, args []string) { + runner.DirectRun(c, args, PluginEntry, map[string]interface{}{ + "connectionId": *connectionId, + "listId": *listId, + }, *timeAfter) + } + runner.RunCmd(cmd) +} diff --git a/backend/plugins/clickup/impl/impl.go b/backend/plugins/clickup/impl/impl.go new file mode 100644 index 00000000000..bdf9b6e7e42 --- /dev/null +++ b/backend/plugins/clickup/impl/impl.go @@ -0,0 +1,225 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "fmt" + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" + "github.com/apache/incubator-devlake/plugins/clickup/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/clickup/tasks" +) + +var _ interface { + plugin.PluginMeta + plugin.PluginInit + plugin.PluginTask + plugin.PluginApi + plugin.PluginModel + plugin.PluginSource + plugin.PluginMigration + plugin.CloseablePluginTask + plugin.DataSourcePluginBlueprintV200 +} = (*ClickUp)(nil) + +type ClickUp struct{} + +func (p ClickUp) Init(basicRes context.BasicRes) errors.Error { + api.Init(basicRes, p) + return nil +} + +func (p ClickUp) Description() string { + return "To collect and enrich data from ClickUp" +} + +func (p ClickUp) Name() string { + return "clickup" +} + +func (p ClickUp) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/clickup" +} + +func (p ClickUp) Connection() dal.Tabler { + return &models.ClickUpConnection{} +} + +func (p ClickUp) Scope() plugin.ToolLayerScope { + return &models.ClickUpFolder{} +} + +func (p ClickUp) ScopeConfig() dal.Tabler { + return &models.ClickUpScopeConfig{} +} + +func (p ClickUp) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.All() +} + +// GetTablesInfo MUST list every model (CI `Test_GetPluginTablesInfo` fails otherwise). +func (p ClickUp) GetTablesInfo() []dal.Tabler { + return []dal.Tabler{ + &models.ClickUpConnection{}, + &models.ClickUpFolder{}, + &models.ClickUpList{}, + &models.ClickUpScopeConfig{}, + &models.ClickUpUser{}, + &models.ClickUpTask{}, + &models.ClickUpTaskComment{}, + } +} + +// SubTaskMetas lists subtasks in dependency order: collect/extract before +// convert; the folder's lists (which classify sprints) before tasks; users +// before tasks (issues reference accounts); the folder-board + sprints before +// board_issues/sprint_issues. +func (p ClickUp) SubTaskMetas() []plugin.SubTaskMeta { + return []plugin.SubTaskMeta{ + tasks.CollectListMeta, + tasks.ExtractListMeta, + tasks.CollectUserMeta, + tasks.ExtractUserMeta, + tasks.CollectTaskMeta, + tasks.ExtractTaskMeta, + tasks.ConvertFolderMeta, + tasks.ConvertSprintMeta, + tasks.ConvertUserMeta, + tasks.ConvertTaskMeta, + } +} + +func (p ClickUp) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + var op tasks.ClickUpOptions + if err := helper.Decode(options, &op, nil); err != nil { + return nil, errors.Default.Wrap(err, "could not decode ClickUp options") + } + if op.ConnectionId == 0 { + return nil, errors.BadInput.New("clickup connectionId is invalid") + } + if op.FolderId == "" { + return nil, errors.BadInput.New("clickup folderId is required") + } + + connection := &models.ClickUpConnection{} + connectionHelper := helper.NewConnectionHelper(taskCtx, nil, p.Name()) + if err := connectionHelper.FirstById(connection, op.ConnectionId); err != nil { + return nil, errors.Default.Wrap(err, "error getting connection for ClickUp plugin") + } + + apiClient, err := tasks.CreateApiClient(taskCtx, connection) + if err != nil { + return nil, errors.Default.Wrap(err, "unable to create ClickUp API client") + } + + // Resolve the scope config. Default to an empty (non-nil) config so subtasks + // can rely on it being present. + scopeConfig := &models.ClickUpScopeConfig{} + if op.ScopeConfigId != 0 { + if err := taskCtx.GetDal().First(scopeConfig, dal.Where("id = ?", op.ScopeConfigId)); err != nil { + return nil, errors.Default.Wrap(err, "error getting scope config for ClickUp plugin") + } + } + + taskData := &tasks.ClickUpTaskData{ + Options: &op, + ApiClient: apiClient, + ScopeConfig: scopeConfig, + } + if op.TimeAfter != "" { + timeAfter, errConv := errors.Convert01(time.Parse(time.RFC3339, op.TimeAfter)) + if errConv != nil { + return nil, errors.BadInput.Wrap(errConv, "invalid timeAfter") + } + taskData.TimeAfter = &timeAfter + } + return taskData, nil +} + +func (p ClickUp) ApiResources() map[string]map[string]plugin.ApiResourceHandler { + return map[string]map[string]plugin.ApiResourceHandler{ + "test": { + "POST": api.TestConnection, + }, + "connections": { + "POST": api.PostConnections, + "GET": api.ListConnections, + }, + "connections/:connectionId": { + "PATCH": api.PatchConnection, + "DELETE": api.DeleteConnection, + "GET": api.GetConnection, + }, + "connections/:connectionId/test": { + "POST": api.TestExistingConnection, + }, + "connections/:connectionId/remote-scopes": { + "GET": api.RemoteScopes, + }, + "connections/:connectionId/proxy/rest/*path": { + "GET": api.Proxy, + }, + "connections/:connectionId/scope-configs": { + "POST": api.PostScopeConfig, + "GET": api.GetScopeConfigList, + }, + "connections/:connectionId/scope-configs/:scopeConfigId": { + "PATCH": api.PatchScopeConfig, + "GET": api.GetScopeConfig, + "DELETE": api.DeleteScopeConfig, + }, + "connections/:connectionId/scopes/:scopeId": { + "GET": api.GetScope, + "PATCH": api.PatchScope, + "DELETE": api.DeleteScope, + }, + "connections/:connectionId/scopes": { + "GET": api.GetScopeList, + "PUT": api.PutScopes, + }, + "scope-config/:scopeConfigId/projects": { + "GET": api.GetProjectsByScopeConfig, + }, + } +} + +func (p ClickUp) MakeDataSourcePipelinePlanV200( + connectionId uint64, + scopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + return api.MakePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes) +} + +func (p ClickUp) Close(taskCtx plugin.TaskContext) errors.Error { + data, ok := taskCtx.GetData().(*tasks.ClickUpTaskData) + if !ok { + return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) + } + if data.ApiClient != nil { + data.ApiClient.Release() + } + return nil +} diff --git a/backend/plugins/clickup/models/connection.go b/backend/plugins/clickup/models/connection.go new file mode 100644 index 00000000000..0f985983966 --- /dev/null +++ b/backend/plugins/clickup/models/connection.go @@ -0,0 +1,74 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +// ClickUpConn holds the essential information to connect to the ClickUp API. +// ClickUp authenticates with a personal API token passed verbatim in the +// `Authorization` header (NO `Bearer` prefix), so we implement our own +// SetupAuthentication instead of reusing helper.AccessToken. OAuth2 is out of +// scope for now. +type ClickUpConn struct { + helper.RestConnection `mapstructure:",squash"` + Token string `mapstructure:"token" validate:"required" json:"token" gorm:"serializer:encdec"` +} + +// SetupAuthentication sets up the HTTP request authentication for the ClickUp API. +func (cc *ClickUpConn) SetupAuthentication(req *http.Request) errors.Error { + req.Header.Set("Authorization", cc.Token) + return nil +} + +func (cc *ClickUpConn) Sanitize() ClickUpConn { + cc.Token = utils.SanitizeString(cc.Token) + return *cc +} + +// ClickUpConnection holds ClickUpConn plus ID/Name for database storage. +type ClickUpConnection struct { + helper.BaseConnection `mapstructure:",squash"` + ClickUpConn `mapstructure:",squash"` +} + +func (connection ClickUpConnection) Sanitize() ClickUpConnection { + connection.ClickUpConn = connection.ClickUpConn.Sanitize() + return connection +} + +func (connection *ClickUpConnection) MergeFromRequest(target *ClickUpConnection, body map[string]interface{}) error { + token := target.Token + if err := helper.DecodeMapStruct(body, target, true); err != nil { + return err + } + modifiedToken := target.Token + if modifiedToken == "" || modifiedToken == utils.SanitizeString(token) { + target.Token = token + } + return nil +} + +func (ClickUpConnection) TableName() string { + return "_tool_clickup_connections" +} diff --git a/backend/plugins/clickup/models/folder.go b/backend/plugins/clickup/models/folder.go new file mode 100644 index 00000000000..347c15e0a21 --- /dev/null +++ b/backend/plugins/clickup/models/folder.go @@ -0,0 +1,72 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.ToolLayerScope = (*ClickUpFolder)(nil) + +// ClickUpFolder is the data-source scope for the ClickUp plugin. A ClickUp +// folder (e.g. a team's "Dev Team" / "Sprint Folder") owns the backlog and the +// rolling sprint lists, so it maps cleanly onto a DevLake domain-layer +// ticket.Board — analogous to a Jira board. Selecting the folder (rather than +// an individual list) means new sprints are picked up automatically on each +// sync and ephemeral/archived sprint lists never need re-scoping. +type ClickUpFolder struct { + common.Scope `mapstructure:",squash"` + FolderId string `json:"folderId" mapstructure:"folderId" gorm:"primaryKey;type:varchar(255)"` + Name string `json:"name" mapstructure:"name" gorm:"type:varchar(255)"` + SpaceId string `json:"spaceId" mapstructure:"spaceId" gorm:"type:varchar(255)"` + SpaceName string `json:"spaceName" mapstructure:"spaceName" gorm:"type:varchar(255)"` +} + +func (f ClickUpFolder) ScopeId() string { + return f.FolderId +} + +func (f ClickUpFolder) ScopeName() string { + return f.Name +} + +func (f ClickUpFolder) ScopeFullName() string { + if f.SpaceName != "" { + return f.SpaceName + "/" + f.Name + } + return f.Name +} + +func (f ClickUpFolder) ScopeParams() interface{} { + return &ClickUpApiParams{ + ConnectionId: f.ConnectionId, + FolderId: f.FolderId, + } +} + +func (ClickUpFolder) TableName() string { + return "_tool_clickup_folders" +} + +// ClickUpApiParams identifies the scope a raw row belongs to. It is stored in +// the `params` column of every _raw_clickup_* table. +type ClickUpApiParams struct { + ConnectionId uint64 + FolderId string +} diff --git a/backend/plugins/clickup/models/list.go b/backend/plugins/clickup/models/list.go new file mode 100644 index 00000000000..6f60ce710e9 --- /dev/null +++ b/backend/plugins/clickup/models/list.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// ClickUpList is a list inside a scoped folder. It is NOT itself a scope (the +// folder is — see ClickUpFolder). A list is one of two things: +// +// - a sprint list (IsSprint) — a rolling sprint, converted to ticket.Sprint; +// tasks in it become sprint_issues. ClickUp teams encode sprints as lists +// named e.g. "v4.3.0 Sprint 40 (7/6/26 - 7/19/26)"; the sprint number and +// start/end dates are parsed from that name (there are no list date fields). +// - a regular list (Backlog / Bug Tracking / DevOps) — its tasks are plain +// board issues, no sprint. +type ClickUpList struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ListId string `gorm:"primaryKey;type:varchar(255)" json:"listId"` + FolderId string `gorm:"index;type:varchar(255)" json:"folderId"` + SpaceId string `gorm:"type:varchar(255)" json:"spaceId"` + Name string `gorm:"type:varchar(255)" json:"name"` + Archived bool `json:"archived"` + IsSprint bool `json:"isSprint"` + SprintName string `gorm:"type:varchar(255)" json:"sprintName"` + StartDate *time.Time `json:"startDate"` + EndDate *time.Time `json:"endDate"` + common.NoPKModel +} + +func (ClickUpList) TableName() string { + return "_tool_clickup_lists" +} diff --git a/backend/plugins/clickup/models/migrationscripts/20260720_add_init_tables.go b/backend/plugins/clickup/models/migrationscripts/20260720_add_init_tables.go new file mode 100644 index 00000000000..24d4beb258e --- /dev/null +++ b/backend/plugins/clickup/models/migrationscripts/20260720_add_init_tables.go @@ -0,0 +1,47 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/clickup/models/migrationscripts/archived" +) + +type addInitTables struct{} + +func (*addInitTables) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &archived.ClickUpConnection{}, + &archived.ClickUpList{}, + &archived.ClickUpScopeConfig{}, + &archived.ClickUpUser{}, + &archived.ClickUpTask{}, + &archived.ClickUpTaskComment{}, + ) +} + +func (*addInitTables) Version() uint64 { + return 20260720000001 +} + +func (*addInitTables) Name() string { + return "clickup init schemas" +} diff --git a/backend/plugins/clickup/models/migrationscripts/20260722_add_folder_scope.go b/backend/plugins/clickup/models/migrationscripts/20260722_add_folder_scope.go new file mode 100644 index 00000000000..3f0affbcbd9 --- /dev/null +++ b/backend/plugins/clickup/models/migrationscripts/20260722_add_folder_scope.go @@ -0,0 +1,102 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +// addFolderScope moves the plugin's data-source scope from List to Folder +// (Jira-parity: folder = board). It creates _tool_clickup_folders and adds the +// sprint/story-point/folder columns to the existing tool tables. AutoMigrate is +// additive: pre-existing columns from the list-scope schema are left in place. + +// frozen snapshots for this migration (suffixed to avoid clashing with the +// 20260720 archived snapshots that share the same table names). + +type clickUpFolder20260722 struct { + archived.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ScopeConfigId uint64 + FolderId string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + SpaceId string `gorm:"type:varchar(255)"` + SpaceName string `gorm:"type:varchar(255)"` +} + +func (clickUpFolder20260722) TableName() string { return "_tool_clickup_folders" } + +type clickUpList20260722 struct { + archived.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ListId string `gorm:"primaryKey;type:varchar(255)"` + FolderId string `gorm:"index;type:varchar(255)"` + SpaceId string `gorm:"type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + Archived bool + IsSprint bool + SprintName string `gorm:"type:varchar(255)"` + StartDate *time.Time + EndDate *time.Time +} + +func (clickUpList20260722) TableName() string { return "_tool_clickup_lists" } + +type clickUpTask20260722 struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + FolderId string `gorm:"index;type:varchar(255)"` + StoryPoint *float64 `gorm:"column:story_point"` + archived.NoPKModel +} + +func (clickUpTask20260722) TableName() string { return "_tool_clickup_tasks" } + +type clickUpScopeConfig20260722 struct { + archived.ScopeConfig + SprintNamePattern string `gorm:"type:varchar(255)"` + StoryPointField string `gorm:"type:varchar(255)"` + DefaultIssueType string `gorm:"type:varchar(100)"` +} + +func (clickUpScopeConfig20260722) TableName() string { return "_tool_clickup_scope_configs" } + +type addFolderScope struct{} + +func (*addFolderScope) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &clickUpFolder20260722{}, + &clickUpList20260722{}, + &clickUpTask20260722{}, + &clickUpScopeConfig20260722{}, + ) +} + +func (*addFolderScope) Version() uint64 { + return 20260722000001 +} + +func (*addFolderScope) Name() string { + return "clickup: add folder scope + sprint/story-point columns" +} diff --git a/backend/plugins/clickup/models/migrationscripts/20260723_add_list_type_patterns.go b/backend/plugins/clickup/models/migrationscripts/20260723_add_list_type_patterns.go new file mode 100644 index 00000000000..28fa313722e --- /dev/null +++ b/backend/plugins/clickup/models/migrationscripts/20260723_add_list_type_patterns.go @@ -0,0 +1,56 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +// addListTypePatterns adds the list-name typing columns to +// _tool_clickup_scope_configs. ClickUp often carries no per-task type, so bugs +// live in a dedicated list (e.g. "QA Bugs"); these patterns let a folder scope +// classify a whole list's tasks as BUG/INCIDENT by list name. AutoMigrate is +// additive. + +type clickUpScopeConfig20260723 struct { + archived.ScopeConfig + BugListPattern string `gorm:"type:varchar(255)"` + IncidentListPattern string `gorm:"type:varchar(255)"` +} + +func (clickUpScopeConfig20260723) TableName() string { return "_tool_clickup_scope_configs" } + +type addListTypePatterns struct{} + +func (*addListTypePatterns) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &clickUpScopeConfig20260723{}, + ) +} + +func (*addListTypePatterns) Version() uint64 { + return 20260723000001 +} + +func (*addListTypePatterns) Name() string { + return "clickup: add bug/incident list-name type patterns to scope config" +} diff --git a/backend/plugins/clickup/models/migrationscripts/archived/models.go b/backend/plugins/clickup/models/migrationscripts/archived/models.go new file mode 100644 index 00000000000..2ecc04f84d5 --- /dev/null +++ b/backend/plugins/clickup/models/migrationscripts/archived/models.go @@ -0,0 +1,113 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package archived holds frozen snapshots of the tool-layer models as they +// existed at each migration. The live models in plugins/clickup/models may +// evolve; these snapshots keep historical migrations stable. +package archived + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +type ClickUpConnection struct { + Name string `gorm:"type:varchar(100);uniqueIndex" json:"name"` + archived.Model + Endpoint string `mapstructure:"endpoint" json:"endpoint"` + Proxy string `mapstructure:"proxy" json:"proxy"` + RateLimitPerHour int `json:"rateLimitPerHour"` + Token string `mapstructure:"token" json:"token" gorm:"serializer:encdec"` +} + +func (ClickUpConnection) TableName() string { return "_tool_clickup_connections" } + +type ClickUpList struct { + archived.NoPKModel + ConnectionId uint64 `json:"connectionId" gorm:"primaryKey"` + ScopeConfigId uint64 `json:"scopeConfigId,omitempty"` + ListId string `json:"listId" gorm:"primaryKey;type:varchar(255)"` + Name string `json:"name" gorm:"type:varchar(255)"` + SpaceId string `json:"spaceId" gorm:"type:varchar(255)"` + SpaceName string `json:"spaceName" gorm:"type:varchar(255)"` +} + +func (ClickUpList) TableName() string { return "_tool_clickup_lists" } + +type ClickUpScopeConfig struct { + archived.ScopeConfig + ConnectionId uint64 `json:"connectionId" gorm:"index"` + Name string `gorm:"type:varchar(255);uniqueIndex" json:"name"` + IssueStatusTodo []string `json:"issueStatusTodo" gorm:"type:json;serializer:json"` + IssueStatusInProgress []string `json:"issueStatusInProgress" gorm:"type:json;serializer:json"` + IssueStatusDone []string `json:"issueStatusDone" gorm:"type:json;serializer:json"` + IssueTypeRequirement string `json:"issueTypeRequirement" gorm:"type:varchar(255)"` + IssueTypeBug string `json:"issueTypeBug" gorm:"type:varchar(255)"` + IssueTypeIncident string `json:"issueTypeIncident" gorm:"type:varchar(255)"` +} + +func (ClickUpScopeConfig) TableName() string { return "_tool_clickup_scope_configs" } + +type ClickUpUser struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + Username string `gorm:"type:varchar(255)"` + Email string `gorm:"type:varchar(255)"` + Color string `gorm:"type:varchar(50)"` + ProfilePicture string `gorm:"type:varchar(255)"` + archived.NoPKModel +} + +func (ClickUpUser) TableName() string { return "_tool_clickup_users" } + +type ClickUpTask struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + ListId string `gorm:"index;type:varchar(255)"` + SpaceId string `gorm:"type:varchar(255)"` + CustomId string `gorm:"type:varchar(255)"` + Name string + Description string + Status string `gorm:"type:varchar(255)"` + StatusType string `gorm:"type:varchar(100)"` + Type string `gorm:"type:varchar(100)"` + Priority string `gorm:"type:varchar(100)"` + Url string `gorm:"type:varchar(255)"` + CreatorId string `gorm:"type:varchar(255)"` + AssigneeId string `gorm:"type:varchar(255)"` + AssigneeName string `gorm:"type:varchar(255)"` + ParentId string `gorm:"type:varchar(255)"` + CreatedDate *time.Time + UpdatedDate *time.Time `gorm:"index"` + ClosedDate *time.Time + archived.NoPKModel +} + +func (ClickUpTask) TableName() string { return "_tool_clickup_tasks" } + +type ClickUpTaskComment struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + TaskId string `gorm:"index;type:varchar(255)"` + Body string + UserId string `gorm:"type:varchar(255)"` + CreatedDate *time.Time + archived.NoPKModel +} + +func (ClickUpTaskComment) TableName() string { return "_tool_clickup_task_comments" } diff --git a/backend/plugins/clickup/models/migrationscripts/register.go b/backend/plugins/clickup/models/migrationscripts/register.go new file mode 100644 index 00000000000..98bff779187 --- /dev/null +++ b/backend/plugins/clickup/models/migrationscripts/register.go @@ -0,0 +1,31 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/plugin" +) + +// All return all the migration scripts +func All() []plugin.MigrationScript { + return []plugin.MigrationScript{ + new(addInitTables), + new(addFolderScope), + new(addListTypePatterns), + } +} diff --git a/backend/plugins/clickup/models/scope_config.go b/backend/plugins/clickup/models/scope_config.go new file mode 100644 index 00000000000..d2ddf62b846 --- /dev/null +++ b/backend/plugins/clickup/models/scope_config.go @@ -0,0 +1,79 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// ClickUpScopeConfig allows a user to override how ClickUp raw statuses and +// task types map onto DevLake's standard domain values. +// +// Status: ClickUp statuses carry a `type` (open/unstarted/custom/done/closed) +// which the plugin maps automatically (open/unstarted -> TODO, custom -> +// IN_PROGRESS, done/closed -> DONE). When any of the IssueStatus* lists below +// are populated, a matching raw status name takes precedence over the +// type-derived default, so teams with bespoke workflows can classify custom +// statuses explicitly. +// +// Type: ClickUp has no native issue "type" on every task, so IssueType* are +// regular expressions matched against a task's derived type string. Precedence +// is INCIDENT > BUG > REQUIREMENT; a task matching none defaults to REQUIREMENT. +// A sensible default (bug -> BUG) is applied by the convertor when no config is +// set. +type ClickUpScopeConfig struct { + common.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"` + IssueStatusTodo []string `mapstructure:"issueStatusTodo,omitempty" json:"issueStatusTodo" gorm:"type:json;serializer:json"` + IssueStatusInProgress []string `mapstructure:"issueStatusInProgress,omitempty" json:"issueStatusInProgress" gorm:"type:json;serializer:json"` + IssueStatusDone []string `mapstructure:"issueStatusDone,omitempty" json:"issueStatusDone" gorm:"type:json;serializer:json"` + IssueTypeRequirement string `mapstructure:"issueTypeRequirement,omitempty" json:"issueTypeRequirement" gorm:"type:varchar(255)"` + IssueTypeBug string `mapstructure:"issueTypeBug,omitempty" json:"issueTypeBug" gorm:"type:varchar(255)"` + IssueTypeIncident string `mapstructure:"issueTypeIncident,omitempty" json:"issueTypeIncident" gorm:"type:varchar(255)"` + // SprintNamePattern is a regex identifying which lists in the folder are + // sprint lists (converted to ticket.Sprint; their tasks become + // sprint_issues). Lists not matching are plain board issues. Empty -> + // defaultSprintNamePattern. + SprintNamePattern string `mapstructure:"sprintNamePattern,omitempty" json:"sprintNamePattern" gorm:"type:varchar(255)"` + // StoryPointField selects where a task's story points come from. Empty + // defaults to ClickUp's native sprint points field ("points"). Set to a + // custom-field name to read Fibonacci/LOE from a custom field instead. + StoryPointField string `mapstructure:"storyPointField,omitempty" json:"storyPointField" gorm:"type:varchar(255)"` + // DefaultIssueType, when set (REQUIREMENT/BUG/INCIDENT), forces every issue + // on this board to that type — used to flag a whole folder as incidents + // (e.g. the "Security Incidents & Response" folder feeding DORA CFR/MTTR). + // Empty -> per-task type detection (IssueType* patterns) applies. + DefaultIssueType string `mapstructure:"defaultIssueType,omitempty" json:"defaultIssueType" gorm:"type:varchar(100)"` + // BugListPattern is a regex matched against a LIST name in the folder; tasks + // in a matching list are classified BUG. ClickUp frequently carries no + // per-task type, so teams group bugs in a dedicated list (e.g. "QA Bugs") + // instead of tagging each task — this types them by list. Empty -> no + // list-based BUG typing. + BugListPattern string `mapstructure:"bugListPattern,omitempty" json:"bugListPattern" gorm:"type:varchar(255)"` + // IncidentListPattern is the INCIDENT counterpart of BugListPattern and + // takes precedence over it for a list matching both. Empty -> none. + IncidentListPattern string `mapstructure:"incidentListPattern,omitempty" json:"incidentListPattern" gorm:"type:varchar(255)"` +} + +func (ClickUpScopeConfig) TableName() string { + return "_tool_clickup_scope_configs" +} + +func (sc *ClickUpScopeConfig) SetConnectionId(c *ClickUpScopeConfig, connectionId uint64) { + c.ConnectionId = connectionId + c.ScopeConfig.ConnectionId = connectionId +} diff --git a/backend/plugins/clickup/models/task.go b/backend/plugins/clickup/models/task.go new file mode 100644 index 00000000000..957889edffb --- /dev/null +++ b/backend/plugins/clickup/models/task.go @@ -0,0 +1,54 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// ClickUpTask is the tool-layer representation of a ClickUp task. +type ClickUpTask struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + ListId string `gorm:"index;type:varchar(255)" json:"listId"` + FolderId string `gorm:"index;type:varchar(255)" json:"folderId"` + SpaceId string `gorm:"type:varchar(255)" json:"spaceId"` + CustomId string `gorm:"type:varchar(255)" json:"customId"` + Name string `json:"name"` + Description string `json:"description"` + Status string `gorm:"type:varchar(255)" json:"status"` + StatusType string `gorm:"type:varchar(100)" json:"statusType"` + Type string `gorm:"type:varchar(100)" json:"type"` + Priority string `gorm:"type:varchar(100)" json:"priority"` + Url string `gorm:"type:varchar(255)" json:"url"` + CreatorId string `gorm:"type:varchar(255)" json:"creatorId"` + AssigneeId string `gorm:"type:varchar(255)" json:"assigneeId"` + AssigneeName string `gorm:"type:varchar(255)" json:"assigneeName"` + ParentId string `gorm:"type:varchar(255)" json:"parentId"` + StoryPoint *float64 `json:"storyPoint"` + CreatedDate *time.Time `json:"createdDate"` + UpdatedDate *time.Time `gorm:"index" json:"updatedDate"` + ClosedDate *time.Time `json:"closedDate"` + common.NoPKModel +} + +func (ClickUpTask) TableName() string { + return "_tool_clickup_tasks" +} diff --git a/backend/plugins/clickup/models/task_comment.go b/backend/plugins/clickup/models/task_comment.go new file mode 100644 index 00000000000..c49d857d42c --- /dev/null +++ b/backend/plugins/clickup/models/task_comment.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// ClickUpTaskComment is the tool-layer representation of a comment on a ClickUp +// task. +// +// TODO(clickup): the comment collector/extractor is not yet implemented. This +// model exists so the table is created up-front and the convertor to +// ticket.IssueComment can be added without a follow-up migration. See +// GET /task/{id}/comment. +type ClickUpTaskComment struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + TaskId string `gorm:"index;type:varchar(255)" json:"taskId"` + Body string `json:"body"` + UserId string `gorm:"type:varchar(255)" json:"userId"` + CreatedDate *time.Time `json:"createdDate"` + common.NoPKModel +} + +func (ClickUpTaskComment) TableName() string { + return "_tool_clickup_task_comments" +} diff --git a/backend/plugins/clickup/models/user.go b/backend/plugins/clickup/models/user.go new file mode 100644 index 00000000000..05ca2aae134 --- /dev/null +++ b/backend/plugins/clickup/models/user.go @@ -0,0 +1,37 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// ClickUpUser is a ClickUp user (tool layer), converted to crossdomain.Account. +type ClickUpUser struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + Username string `gorm:"type:varchar(255)" json:"username"` + Email string `gorm:"type:varchar(255)" json:"email"` + Color string `gorm:"type:varchar(50)" json:"color"` + ProfilePicture string `gorm:"type:varchar(255)" json:"profilePicture"` + common.NoPKModel +} + +func (ClickUpUser) TableName() string { + return "_tool_clickup_users" +} diff --git a/backend/plugins/clickup/tasks/api_client.go b/backend/plugins/clickup/tasks/api_client.go new file mode 100644 index 00000000000..85333cd184e --- /dev/null +++ b/backend/plugins/clickup/tasks/api_client.go @@ -0,0 +1,47 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +// DefaultEndpoint is the ClickUp REST v2 base URL. Note the trailing slash: +// devlake joins it with relative UrlTemplates. +const DefaultEndpoint = "https://api.clickup.com/api/v2/" + +// CreateApiClient creates a new rate-limited async API client for ClickUp. +func CreateApiClient(taskCtx plugin.TaskContext, connection *models.ClickUpConnection) (*api.ApiAsyncClient, errors.Error) { + if connection.Endpoint == "" { + connection.Endpoint = DefaultEndpoint + } + apiClient, err := api.NewApiClientFromConnection(taskCtx.GetContext(), taskCtx, connection) + if err != nil { + return nil, err + } + + asyncApiClient, err := api.CreateAsyncApiClient(taskCtx, apiClient, nil) + if err != nil { + return nil, err + } + + return asyncApiClient, nil +} diff --git a/backend/plugins/clickup/tasks/folder_convertor.go b/backend/plugins/clickup/tasks/folder_convertor.go new file mode 100644 index 00000000000..e5306ef9ca4 --- /dev/null +++ b/backend/plugins/clickup/tasks/folder_convertor.go @@ -0,0 +1,93 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +// RAW_FOLDER_TABLE labels the raw-data lineage for the folder-scope-derived +// board. The folder is added as a scope (no collector), so this is a logical +// tag only. +const RAW_FOLDER_TABLE = "clickup_folders" + +var ConvertFolderMeta = plugin.SubTaskMeta{ + Name: "Convert Folders", + EntryPoint: ConvertFolders, + EnabledByDefault: true, + Description: "Convert the ClickUp folder scope (_tool_clickup_folders) into the domain layer table boards", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.ClickUpFolder{}.TableName()}, + ProductTables: []string{ticket.Board{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertFolders + +func ConvertFolders(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*ClickUpTaskData) + connectionId := data.Options.ConnectionId + + // boardId must be generated identically to the task/sprint convertors so the + // board joins to the board_issues/board_sprints that reference it. + boardIdGen := didgen.NewDomainIdGenerator(&models.ClickUpFolder{}) + + cursor, err := db.Cursor( + dal.From(&models.ClickUpFolder{}), + dal.Where("connection_id = ? AND folder_id = ?", connectionId, data.Options.FolderId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: ClickUpApiParams{ + ConnectionId: connectionId, + FolderId: data.Options.FolderId, + }, + Table: RAW_FOLDER_TABLE, + }, + InputRowType: reflect.TypeOf(models.ClickUpFolder{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + folder := inputRow.(*models.ClickUpFolder) + board := &ticket.Board{ + DomainEntity: domainlayer.DomainEntity{Id: boardIdGen.Generate(connectionId, folder.FolderId)}, + Name: folder.ScopeFullName(), + Type: "clickup", + } + return []interface{}{board}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/clickup/tasks/list_collector.go b/backend/plugins/clickup/tasks/list_collector.go new file mode 100644 index 00000000000..bc601f68218 --- /dev/null +++ b/backend/plugins/clickup/tasks/list_collector.go @@ -0,0 +1,111 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + "net/url" + "strconv" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +// RAW_LIST_TABLE holds the raw lists of a scoped folder (GET /folder/{id}/list). +const RAW_LIST_TABLE = "clickup_lists" + +// clickUpListResponse mirrors GET /folder/{id}/list. The endpoint is not +// paginated and returns the folder's lists under `lists`. +type clickUpListResponse struct { + Lists []json.RawMessage `json:"lists"` +} + +// archivedInput drives the collector to request the folder's lists twice: once +// for active lists and once for archived lists (ClickUp's /folder/{id}/list +// returns active by default and archived only when archived=true). +type archivedInput struct { + Archived bool +} + +// archivedIterator yields {false, true} so a single Execute collects both the +// active and the archived lists into the same raw table. +type archivedIterator struct { + vals []bool + i int +} + +func newArchivedIterator() *archivedIterator { return &archivedIterator{vals: []bool{false, true}} } + +func (it *archivedIterator) HasNext() bool { return it.i < len(it.vals) } + +func (it *archivedIterator) Fetch() (interface{}, errors.Error) { + v := it.vals[it.i] + it.i++ + return &archivedInput{Archived: v}, nil +} + +func (it *archivedIterator) Close() errors.Error { return nil } + +var CollectListMeta = plugin.SubTaskMeta{ + Name: "Collect Lists", + EntryPoint: CollectLists, + EnabledByDefault: true, + Description: "Collect the lists (backlog + sprint lists, active and archived) of a scoped ClickUp folder", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = CollectLists + +func CollectLists(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*ClickUpTaskData) + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: ClickUpApiParams{ + ConnectionId: data.Options.ConnectionId, + FolderId: data.Options.FolderId, + }, + Table: RAW_LIST_TABLE, + }, + ApiClient: data.ApiClient, + Input: newArchivedIterator(), + UrlTemplate: "folder/{{ .Params.FolderId }}/list", + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + archived := false + if in, ok := reqData.Input.(*archivedInput); ok { + archived = in.Archived + } + query.Set("archived", strconv.FormatBool(archived)) + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var resp clickUpListResponse + if err := api.UnmarshalResponse(res, &resp); err != nil { + return nil, err + } + return resp.Lists, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/clickup/tasks/list_extractor.go b/backend/plugins/clickup/tasks/list_extractor.go new file mode 100644 index 00000000000..01d18563e45 --- /dev/null +++ b/backend/plugins/clickup/tasks/list_extractor.go @@ -0,0 +1,102 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +var ExtractListMeta = plugin.SubTaskMeta{ + Name: "Extract Lists", + EntryPoint: ExtractLists, + EnabledByDefault: true, + Description: "Extract raw folder lists into _tool_clickup_lists, classifying sprint lists", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractLists + +// ClickUpApiList is the subset of a ClickUp list JSON the extractor reads. +type ClickUpApiList struct { + Id string `json:"id"` + Name string `json:"name"` + Archived bool `json:"archived"` + Folder *struct { + Id string `json:"id"` + } `json:"folder"` + Space *struct { + Id string `json:"id"` + } `json:"space"` +} + +func ExtractLists(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*ClickUpTaskData) + detector, err := newSprintDetector(data.ScopeConfig) + if err != nil { + return err + } + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: ClickUpApiParams{ + ConnectionId: data.Options.ConnectionId, + FolderId: data.Options.FolderId, + }, + Table: RAW_LIST_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiList := &ClickUpApiList{} + if err := errors.Convert(json.Unmarshal(row.Data, apiList)); err != nil { + return nil, err + } + if apiList.Id == "" { + return nil, nil + } + folderId := data.Options.FolderId + if apiList.Folder != nil && apiList.Folder.Id != "" { + folderId = apiList.Folder.Id + } + list := &models.ClickUpList{ + ConnectionId: data.Options.ConnectionId, + ListId: apiList.Id, + FolderId: folderId, + Name: apiList.Name, + Archived: apiList.Archived, + } + if apiList.Space != nil { + list.SpaceId = apiList.Space.Id + } + if sprint := detector.detect(apiList.Name); sprint != nil { + list.IsSprint = true + list.SprintName = sprint.name + list.StartDate = sprint.start + list.EndDate = sprint.end + } + return []interface{}{list}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/clickup/tasks/shared.go b/backend/plugins/clickup/tasks/shared.go new file mode 100644 index 00000000000..783c3c6e79f --- /dev/null +++ b/backend/plugins/clickup/tasks/shared.go @@ -0,0 +1,254 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "regexp" + "strconv" + "strings" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +// parseClickUpTime parses a ClickUp millisecond-epoch timestamp. ClickUp encodes +// timestamps as strings of milliseconds since the Unix epoch (e.g. "1567780450202"). +// It returns nil for empty / zero / unparseable values so callers can leave the +// corresponding *time.Time unset rather than storing a bogus 1970 date. +func parseClickUpTime(ms string) *time.Time { + ms = strings.TrimSpace(ms) + if ms == "" { + return nil + } + millis, err := strconv.ParseInt(ms, 10, 64) + if err != nil || millis <= 0 { + return nil + } + t := time.UnixMilli(millis).UTC() + return &t +} + +// defaultSprintNamePattern identifies sprint lists by name. ClickUp teams name +// sprint lists like "v4.3.0 Sprint 40 (7/6/26 - 7/19/26)" or "Sprint 31 (7/6 - +// 7/19)", so any list whose name contains "Sprint " is treated as a sprint. +const defaultSprintNamePattern = `(?i)sprint\s*\d+` + +// sprintDateRange captures the "(start - end)" span embedded in a sprint list +// name. Group 1 = start token, group 2 = end token (each m/d[/yy] or d/m[/yy]). +var sprintDateRange = regexp.MustCompile(`\(\s*([\d/]+)\s*-\s*([\d/]+)\s*\)`) + +type sprintInfo struct { + name string + start *time.Time + end *time.Time +} + +// sprintDetector classifies a list name as a sprint (or not) using the scope +// config's SprintNamePattern (default defaultSprintNamePattern). +type sprintDetector struct { + re *regexp.Regexp +} + +func newSprintDetector(sc *models.ClickUpScopeConfig) (*sprintDetector, errors.Error) { + pattern := defaultSprintNamePattern + if sc != nil && sc.SprintNamePattern != "" { + pattern = sc.SprintNamePattern + } + re, err := errors.Convert01(regexp.Compile(pattern)) + if err != nil { + return nil, errors.Default.Wrap(err, "invalid sprintNamePattern") + } + return &sprintDetector{re: re}, nil +} + +// detect returns sprint info for a matching list name, or nil for a non-sprint +// (backlog / bug / other) list. +func (d *sprintDetector) detect(name string) *sprintInfo { + if !d.re.MatchString(name) { + return nil + } + start, end := parseSprintDates(name) + return &sprintInfo{name: name, start: start, end: end} +} + +// parseSprintDates extracts the start/end dates from a sprint list name's +// "(m/d/yy - m/d/yy)" span. Teams differ on ordering (Toto uses M/D/YY, +// Blockchain D/M/YY), so the order is disambiguated per token: a component > 12 +// must be the day. Genuinely ambiguous tokens (both <= 12) default to M/D. +// Tokens without a year are left unset (nil) rather than guessed. +func parseSprintDates(name string) (*time.Time, *time.Time) { + m := sprintDateRange.FindStringSubmatch(name) + if m == nil { + return nil, nil + } + return parseSprintDate(m[1]), parseSprintDate(m[2]) +} + +func parseSprintDate(token string) *time.Time { + parts := strings.Split(strings.TrimSpace(token), "/") + if len(parts) != 3 { + // No year component -> cannot form a reliable date. + return nil + } + a, errA := strconv.Atoi(parts[0]) + b, errB := strconv.Atoi(parts[1]) + y, errY := strconv.Atoi(parts[2]) + if errA != nil || errB != nil || errY != nil { + return nil + } + month, day := a, b + if a > 12 { // first component can't be a month -> D/M ordering + month, day = b, a + } + if month < 1 || month > 12 || day < 1 || day > 31 { + return nil + } + if y < 100 { + y += 2000 + } + t := time.Date(y, time.Month(month), day, 0, 0, 0, 0, time.UTC) + return &t +} + +// statusFromType maps a ClickUp status.type onto a DevLake standard issue +// status. ClickUp's status types are standardized: +// +// open, unstarted -> TODO +// custom -> IN_PROGRESS +// done, closed -> DONE +// +// Any unrecognized type falls back to OTHER so unexpected API values surface +// rather than silently masquerading as a known status. +func statusFromType(statusType string) string { + switch strings.ToLower(statusType) { + case "open", "unstarted": + return ticket.TODO + case "custom": + return ticket.IN_PROGRESS + case "done", "closed": + return ticket.DONE + default: + return ticket.OTHER + } +} + +// statusMapper resolves a ClickUp task's domain status. When the scope config +// supplies explicit status name lists, a matching raw status name wins; +// otherwise the status.type-derived default is used. +type statusMapper struct { + byName map[string]string + hasList bool +} + +func newStatusMapper(sc *models.ClickUpScopeConfig) *statusMapper { + m := &statusMapper{byName: map[string]string{}} + if sc == nil { + return m + } + add := func(names []string, domain string) { + for _, n := range names { + n = strings.ToLower(strings.TrimSpace(n)) + if n != "" { + m.byName[n] = domain + m.hasList = true + } + } + } + add(sc.IssueStatusTodo, ticket.TODO) + add(sc.IssueStatusInProgress, ticket.IN_PROGRESS) + add(sc.IssueStatusDone, ticket.DONE) + return m +} + +// statusOf returns the domain status for a raw status name / type. A user-configured +// name mapping takes precedence over the type-derived default. +func (m *statusMapper) statusOf(rawStatus, statusType string) string { + if m.hasList { + if domain, ok := m.byName[strings.ToLower(strings.TrimSpace(rawStatus))]; ok { + return domain + } + } + return statusFromType(statusType) +} + +// issueTypeMatcher derives the domain ticket.Issue.Type from a task's derived +// type string using the scope config's regex patterns. Precedence is +// INCIDENT > BUG > REQUIREMENT; a task matching none defaults to REQUIREMENT. +// +// When no patterns are configured a sensible default (bug -> BUG) is applied so +// bug-typed tasks feed DORA change-failure-rate out of the box. +type issueTypeMatcher struct { + incident *regexp.Regexp + bug *regexp.Regexp + requirement *regexp.Regexp +} + +// defaultBugPattern matches ClickUp's built-in "Bug" custom task type name +// (case-insensitive) when the scope config leaves IssueTypeBug empty. +const defaultBugPattern = "(?i)^bug$" + +func newIssueTypeMatcher(sc *models.ClickUpScopeConfig) (*issueTypeMatcher, errors.Error) { + m := &issueTypeMatcher{} + bugPattern := defaultBugPattern + var incidentPattern, requirementPattern string + if sc != nil { + if sc.IssueTypeBug != "" { + bugPattern = sc.IssueTypeBug + } + incidentPattern = sc.IssueTypeIncident + requirementPattern = sc.IssueTypeRequirement + } + for _, p := range []struct { + pattern string + field string + out **regexp.Regexp + }{ + {incidentPattern, "issueTypeIncident", &m.incident}, + {bugPattern, "issueTypeBug", &m.bug}, + {requirementPattern, "issueTypeRequirement", &m.requirement}, + } { + if p.pattern == "" { + continue + } + re, err := errors.Convert01(regexp.Compile(p.pattern)) + if err != nil { + return nil, errors.Default.Wrap(err, "invalid "+p.field+" pattern") + } + *p.out = re + } + return m, nil +} + +// typeOf returns the domain issue type for a task's derived type string. +func (m *issueTypeMatcher) typeOf(taskType string) string { + for _, c := range []struct { + pattern *regexp.Regexp + typ string + }{ + {m.incident, ticket.INCIDENT}, + {m.bug, ticket.BUG}, + {m.requirement, ticket.REQUIREMENT}, + } { + if c.pattern != nil && c.pattern.MatchString(taskType) { + return c.typ + } + } + return ticket.REQUIREMENT +} diff --git a/backend/plugins/clickup/tasks/shared_test.go b/backend/plugins/clickup/tasks/shared_test.go new file mode 100644 index 00000000000..a84fb1a693b --- /dev/null +++ b/backend/plugins/clickup/tasks/shared_test.go @@ -0,0 +1,254 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "testing" + + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +func TestParseSprintDate(t *testing.T) { + tests := []struct { + name string + token string + want string // "" means nil (no reliable date) + }{ + {"Toto M/D/YY", "7/6/26", "2026-07-06"}, + {"Toto M/D/YY day>12", "7/19/26", "2026-07-19"}, + {"Blockchain D/M/YY day>12", "29/6/26", "2026-06-29"}, + {"Blockchain D/M/YY", "26/7/26", "2026-07-26"}, + {"ambiguous both<=12 defaults M/D", "3/4/26", "2026-03-04"}, + {"no year -> nil", "7/6", ""}, + {"empty -> nil", "", ""}, + {"invalid month both>12 -> nil", "13/13/26", ""}, + {"non-numeric -> nil", "a/b/c", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := parseSprintDate(tt.token) + if tt.want == "" { + if got != nil { + t.Fatalf("expected nil, got %v", got) + } + return + } + if got == nil { + t.Fatalf("expected %s, got nil", tt.want) + } + if g := got.Format("2006-01-02"); g != tt.want { + t.Fatalf("expected %s, got %s", tt.want, g) + } + }) + } +} + +func TestSprintDetector(t *testing.T) { + det, err := newSprintDetector(nil) // nil -> default pattern + if err != nil { + t.Fatalf("newSprintDetector: %v", err) + } + tests := []struct { + name string + listName string + wantSprint bool + wantStart string + }{ + {"toto sprint with dates", "v4.3.0 Sprint 40 (7/6/26 - 7/19/26)", true, "2026-07-06"}, + {"blockchain sprint D/M", "Sprint 25 (29/6/26 - 26/7/26)", true, "2026-06-29"}, + {"sprint no year -> no dates", "Sprint 31 (7/6 - 7/19)", true, ""}, + {"backlog is not a sprint", "Backlog", false, ""}, + {"qa bugs is not a sprint", "QA Bugs", false, ""}, + {"devops is not a sprint", "DevOps", false, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := det.detect(tt.listName) + if tt.wantSprint != (got != nil) { + t.Fatalf("wantSprint=%v, got=%v", tt.wantSprint, got != nil) + } + if got == nil { + return + } + if got.name != tt.listName { + t.Fatalf("sprint name = %q, want %q", got.name, tt.listName) + } + if tt.wantStart == "" { + if got.start != nil { + t.Fatalf("expected nil start, got %v", got.start) + } + } else if got.start == nil || got.start.Format("2006-01-02") != tt.wantStart { + t.Fatalf("start = %v, want %s", got.start, tt.wantStart) + } + }) + } +} + +func TestStatusFromType(t *testing.T) { + tests := []struct { + statusType string + want string + }{ + {"open", ticket.TODO}, + {"unstarted", ticket.TODO}, + {"custom", ticket.IN_PROGRESS}, + {"done", ticket.DONE}, + {"closed", ticket.DONE}, + {"CLOSED", ticket.DONE}, // case-insensitive + {"mystery", ticket.OTHER}, + } + for _, tt := range tests { + t.Run(tt.statusType, func(t *testing.T) { + if got := statusFromType(tt.statusType); got != tt.want { + t.Fatalf("statusFromType(%q) = %q, want %q", tt.statusType, got, tt.want) + } + }) + } +} + +func TestStatusMapperOverride(t *testing.T) { + // Toto tags "to do" and "re-open" as ClickUp type "custom", which the + // type-default would file as IN_PROGRESS. The scope-config override must win. + sc := &models.ClickUpScopeConfig{ + IssueStatusTodo: []string{"to do", "re-open"}, + IssueStatusInProgress: []string{"in development"}, + IssueStatusDone: []string{"deployed", "Closed"}, + } + m := newStatusMapper(sc) + tests := []struct { + rawStatus string + statusType string + want string + }{ + {"to do", "custom", ticket.TODO}, // override beats type default + {"re-open", "custom", ticket.TODO}, // override beats type default + {"in development", "custom", ticket.IN_PROGRESS}, + {"deployed", "done", ticket.DONE}, + {"CLOSED", "closed", ticket.DONE}, // override match is case-insensitive + {"in code review", "custom", ticket.IN_PROGRESS}, // not listed -> type default + } + for _, tt := range tests { + t.Run(tt.rawStatus, func(t *testing.T) { + if got := m.statusOf(tt.rawStatus, tt.statusType); got != tt.want { + t.Fatalf("statusOf(%q,%q) = %q, want %q", tt.rawStatus, tt.statusType, got, tt.want) + } + }) + } +} + +func TestStatusMapperNoConfigFallsBackToType(t *testing.T) { + m := newStatusMapper(nil) + if got := m.statusOf("anything", "open"); got != ticket.TODO { + t.Fatalf("expected TODO from type, got %q", got) + } +} + +func TestIssueTypeMatcher(t *testing.T) { + sc := &models.ClickUpScopeConfig{ + IssueTypeIncident: "(?i)incident", + IssueTypeBug: "(?i)^bug$", + IssueTypeRequirement: "(?i)(feature|story)", + } + m, err := newIssueTypeMatcher(sc) + if err != nil { + t.Fatalf("newIssueTypeMatcher: %v", err) + } + tests := []struct { + taskType string + want string + }{ + {"incident", ticket.INCIDENT}, + {"Bug", ticket.BUG}, + {"feature", ticket.REQUIREMENT}, + {"", ticket.REQUIREMENT}, // no match -> default REQUIREMENT + } + for _, tt := range tests { + t.Run(tt.taskType, func(t *testing.T) { + if got := m.typeOf(tt.taskType); got != tt.want { + t.Fatalf("typeOf(%q) = %q, want %q", tt.taskType, got, tt.want) + } + }) + } +} + +func TestIssueTypeMatcherDefaultBug(t *testing.T) { + // nil config still classifies ClickUp's built-in "Bug" via defaultBugPattern. + m, err := newIssueTypeMatcher(nil) + if err != nil { + t.Fatalf("newIssueTypeMatcher: %v", err) + } + if got := m.typeOf("bug"); got != ticket.BUG { + t.Fatalf("typeOf(bug) = %q, want BUG", got) + } + if got := m.typeOf("anything-else"); got != ticket.REQUIREMENT { + t.Fatalf("typeOf(anything-else) = %q, want REQUIREMENT", got) + } +} + +func TestStoryPointOf(t *testing.T) { + fp := func(f float64) *float64 { return &f } + native := &ClickUpApiTask{Points: fp(8)} + // default (no field configured) uses native points + if got := storyPointOf(native, nil); got == nil || *got != 8 { + t.Fatalf("native points: got %v, want 8", got) + } + // configured custom field wins + custom := &ClickUpApiTask{ + Points: fp(8), + CustomFields: []clickUpCustomField{ + {Name: "LOE", Value: json.RawMessage(`"13"`)}, + }, + } + sc := &models.ClickUpScopeConfig{StoryPointField: "loe"} // case-insensitive + if got := storyPointOf(custom, sc); got == nil || *got != 13 { + t.Fatalf("custom field: got %v, want 13", got) + } + // configured field absent on task -> nil (does not fall back to native) + if got := storyPointOf(native, sc); got != nil { + t.Fatalf("missing custom field: got %v, want nil", got) + } +} + +func TestNumericValue(t *testing.T) { + tests := []struct { + raw string + want *float64 + }{ + {`5`, ptr(5)}, + {`"8"`, ptr(8)}, + {`3.5`, ptr(3.5)}, + {``, nil}, + {`null`, nil}, + {`"abc"`, nil}, + } + for _, tt := range tests { + t.Run(tt.raw, func(t *testing.T) { + got := numericValue(json.RawMessage(tt.raw)) + if (tt.want == nil) != (got == nil) { + t.Fatalf("raw %q: got %v, want %v", tt.raw, got, tt.want) + } + if tt.want != nil && *got != *tt.want { + t.Fatalf("raw %q: got %v, want %v", tt.raw, *got, *tt.want) + } + }) + } +} + +func ptr(f float64) *float64 { return &f } diff --git a/backend/plugins/clickup/tasks/sprint_convertor.go b/backend/plugins/clickup/tasks/sprint_convertor.go new file mode 100644 index 00000000000..b5849882e3c --- /dev/null +++ b/backend/plugins/clickup/tasks/sprint_convertor.go @@ -0,0 +1,124 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +// Sprint status values follow the convention used by the Jira plugin. +const ( + sprintStatusFuture = "FUTURE" + sprintStatusActive = "ACTIVE" + sprintStatusClosed = "CLOSED" +) + +var ConvertSprintMeta = plugin.SubTaskMeta{ + Name: "Convert Sprints", + EntryPoint: ConvertSprints, + EnabledByDefault: true, + Description: "Convert sprint lists (_tool_clickup_lists where is_sprint) into domain tables sprints and board_sprints", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.ClickUpList{}.TableName()}, + ProductTables: []string{ticket.Sprint{}.TableName(), ticket.BoardSprint{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertSprints + +func ConvertSprints(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*ClickUpTaskData) + connectionId := data.Options.ConnectionId + + sprintIdGen := didgen.NewDomainIdGenerator(&models.ClickUpList{}) + boardIdGen := didgen.NewDomainIdGenerator(&models.ClickUpFolder{}) + boardId := boardIdGen.Generate(connectionId, data.Options.FolderId) + now := time.Now() + + cursor, err := db.Cursor( + dal.From(&models.ClickUpList{}), + dal.Where("connection_id = ? AND folder_id = ? AND is_sprint = ?", connectionId, data.Options.FolderId, true), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: ClickUpApiParams{ + ConnectionId: connectionId, + FolderId: data.Options.FolderId, + }, + Table: RAW_LIST_TABLE, + }, + InputRowType: reflect.TypeOf(models.ClickUpList{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + list := inputRow.(*models.ClickUpList) + sprintId := sprintIdGen.Generate(connectionId, list.ListId) + sprint := &ticket.Sprint{ + DomainEntity: domainlayer.DomainEntity{Id: sprintId}, + Name: list.SprintName, + Status: sprintStatus(list, now), + StartedDate: list.StartDate, + EndedDate: list.EndDate, + OriginalBoardID: boardId, + } + if sprint.Status == sprintStatusClosed { + sprint.CompletedDate = list.EndDate + } + boardSprint := &ticket.BoardSprint{ + BoardId: boardId, + SprintId: sprintId, + } + return []interface{}{sprint, boardSprint}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} + +// sprintStatus derives a sprint's lifecycle from its dates and archived flag. +// Archived sprint lists are always closed; otherwise the current time relative +// to the parsed start/end window decides. Missing dates default to ACTIVE. +func sprintStatus(list *models.ClickUpList, now time.Time) string { + if list.Archived { + return sprintStatusClosed + } + if list.EndDate != nil && now.After(*list.EndDate) { + return sprintStatusClosed + } + if list.StartDate != nil && now.Before(*list.StartDate) { + return sprintStatusFuture + } + return sprintStatusActive +} diff --git a/backend/plugins/clickup/tasks/task_collector.go b/backend/plugins/clickup/tasks/task_collector.go new file mode 100644 index 00000000000..8d2572e5f73 --- /dev/null +++ b/backend/plugins/clickup/tasks/task_collector.go @@ -0,0 +1,137 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + "net/url" + "reflect" + "strconv" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +const RAW_TASK_TABLE = "clickup_tasks" + +// clickUpTaskListResponse mirrors the envelope returned by GET /list/{id}/task. +// ClickUp uses zero-based page numbers and signals the end of the collection +// with `last_page: true` (and/or an empty `tasks` array). +type clickUpTaskListResponse struct { + Tasks []json.RawMessage `json:"tasks"` + LastPage bool `json:"last_page"` +} + +// listInput is the per-list iterator element driving task collection: tasks are +// collected list-by-list across every list in the scoped folder. +type listInput struct { + ListId string `gorm:"column:list_id"` +} + +var CollectTaskMeta = plugin.SubTaskMeta{ + Name: "Collect Tasks", + EntryPoint: CollectTasks, + EnabledByDefault: true, + Description: "Collect tasks for every list in the scoped ClickUp folder (page-based pagination)", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = CollectTasks + +func CollectTasks(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*ClickUpTaskData) + db := taskCtx.GetDal() + + // Iterate every list of this folder collected by CollectLists. + cursor, err := db.Cursor( + dal.Select("list_id"), + dal.From(&models.ClickUpList{}), + dal.Where("connection_id = ? AND folder_id = ?", data.Options.ConnectionId, data.Options.FolderId), + ) + if err != nil { + return err + } + iterator, err := api.NewDalCursorIterator(db, cursor, reflect.TypeOf(listInput{})) + if err != nil { + return err + } + + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: ClickUpApiParams{ + ConnectionId: data.Options.ConnectionId, + FolderId: data.Options.FolderId, + }, + Table: RAW_TASK_TABLE, + }, + ApiClient: data.ApiClient, + Input: iterator, + PageSize: 100, + UrlTemplate: "list/{{ .Input.ListId }}/task", + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("subtasks", "true") + query.Set("include_closed", "true") + page := "0" + if reqData.CustomData != nil { + if p, ok := reqData.CustomData.(string); ok && p != "" { + page = p + } + } + query.Set("page", page) + // Incremental collection: restrict to tasks updated after the + // configured cut-off (ClickUp expects milliseconds since epoch). + if data.TimeAfter != nil { + query.Set("date_updated_gt", strconv.FormatInt(data.TimeAfter.UnixMilli(), 10)) + } + return query, nil + }, + GetNextPageCustomData: func(prevReqData *api.RequestData, prevPageResponse *http.Response) (interface{}, errors.Error) { + var resp clickUpTaskListResponse + if err := api.UnmarshalResponse(prevPageResponse, &resp); err != nil { + return nil, err + } + if resp.LastPage || len(resp.Tasks) == 0 { + return nil, api.ErrFinishCollect + } + prevPage := 0 + if prevReqData.CustomData != nil { + if p, ok := prevReqData.CustomData.(string); ok { + prevPage, _ = strconv.Atoi(p) + } + } + return strconv.Itoa(prevPage + 1), nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var resp clickUpTaskListResponse + if err := api.UnmarshalResponse(res, &resp); err != nil { + return nil, err + } + return resp.Tasks, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/clickup/tasks/task_convertor.go b/backend/plugins/clickup/tasks/task_convertor.go new file mode 100644 index 00000000000..7ceeb608798 --- /dev/null +++ b/backend/plugins/clickup/tasks/task_convertor.go @@ -0,0 +1,254 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + "regexp" + "strings" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +var ConvertTaskMeta = plugin.SubTaskMeta{ + Name: "Convert Tasks", + EntryPoint: ConvertTasks, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_clickup_tasks into domain layer tables issues, board_issues and sprint_issues", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.ClickUpTask{}.TableName(), models.ClickUpList{}.TableName(), RAW_TASK_TABLE}, + ProductTables: []string{ticket.Issue{}.TableName(), ticket.BoardIssue{}.TableName(), ticket.SprintIssue{}.TableName(), ticket.IssueAssignee{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertTasks + +func ConvertTasks(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*ClickUpTaskData) + connectionId := data.Options.ConnectionId + + issueIdGen := didgen.NewDomainIdGenerator(&models.ClickUpTask{}) + accountIdGen := didgen.NewDomainIdGenerator(&models.ClickUpUser{}) + boardIdGen := didgen.NewDomainIdGenerator(&models.ClickUpFolder{}) + sprintIdGen := didgen.NewDomainIdGenerator(&models.ClickUpList{}) + boardId := boardIdGen.Generate(connectionId, data.Options.FolderId) + + // Set of sprint list ids so a task in a sprint list also produces a + // sprint_issue (velocity/throughput). Non-sprint lists (Backlog / Bug + // Tracking) contribute only board_issues. + sprintListIds, err := loadSprintListIds(db, connectionId, data.Options.FolderId) + if err != nil { + return err + } + + // listTypes maps a list id -> forced issue type (BUG/INCIDENT) when the + // list name matches the scope-config's Bug/IncidentListPattern. ClickUp + // tasks often carry no per-task type, so bugs are grouped in a list (e.g. + // "QA Bugs") rather than tagged; this types them by list. + listTypes, err := loadListTypeOverrides(db, connectionId, data.Options.FolderId, data.ScopeConfig) + if err != nil { + return err + } + + statusMapper := newStatusMapper(data.ScopeConfig) + typeMatcher, err := newIssueTypeMatcher(data.ScopeConfig) + if err != nil { + return err + } + defaultType := "" + if data.ScopeConfig != nil && data.ScopeConfig.DefaultIssueType != "" { + defaultType = strings.ToUpper(strings.TrimSpace(data.ScopeConfig.DefaultIssueType)) + } + + cursor, err := db.Cursor( + dal.From(&models.ClickUpTask{}), + dal.Where("connection_id = ? AND folder_id = ?", connectionId, data.Options.FolderId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: ClickUpApiParams{ + ConnectionId: connectionId, + FolderId: data.Options.FolderId, + }, + Table: RAW_TASK_TABLE, + }, + InputRowType: reflect.TypeOf(models.ClickUpTask{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + task := inputRow.(*models.ClickUpTask) + + issueKey := task.CustomId + if issueKey == "" { + issueKey = task.Id + } + + // Precedence: forced DefaultIssueType > list-name pattern > + // per-task type detection. + issueType := typeMatcher.typeOf(task.Type) + if lt, ok := listTypes[task.ListId]; ok { + issueType = lt + } + if defaultType != "" { + issueType = defaultType + } + + domainIssue := &ticket.Issue{ + DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(connectionId, task.Id)}, + IssueKey: issueKey, + Title: task.Name, + Description: task.Description, + Url: task.Url, + Type: issueType, + OriginalType: task.Type, + Status: statusMapper.statusOf(task.Status, task.StatusType), + OriginalStatus: task.Status, + Priority: task.Priority, + StoryPoint: task.StoryPoint, + CreatedDate: task.CreatedDate, + UpdatedDate: task.UpdatedDate, + ResolutionDate: task.ClosedDate, + } + if task.CreatorId != "" { + domainIssue.CreatorId = accountIdGen.Generate(connectionId, task.CreatorId) + } + if task.AssigneeId != "" { + domainIssue.AssigneeId = accountIdGen.Generate(connectionId, task.AssigneeId) + domainIssue.AssigneeName = task.AssigneeName + } + if task.ParentId != "" { + domainIssue.ParentIssueId = issueIdGen.Generate(connectionId, task.ParentId) + domainIssue.IsSubtask = true + } + // Fallback lead time. Guard against a resolution that precedes + // creation (clock skew / imported tasks): a negative duration cast + // to uint yields garbage, so leave lead time unset instead. + if domainIssue.ResolutionDate != nil && task.CreatedDate != nil && + domainIssue.ResolutionDate.After(*task.CreatedDate) { + minutes := uint(domainIssue.ResolutionDate.Sub(*task.CreatedDate).Minutes()) + domainIssue.LeadTimeMinutes = &minutes + } + + results := []interface{}{ + domainIssue, + &ticket.BoardIssue{BoardId: boardId, IssueId: domainIssue.Id}, + } + if task.ListId != "" && sprintListIds[task.ListId] { + results = append(results, &ticket.SprintIssue{ + SprintId: sprintIdGen.Generate(connectionId, task.ListId), + IssueId: domainIssue.Id, + }) + } + if domainIssue.AssigneeId != "" { + results = append(results, &ticket.IssueAssignee{ + IssueId: domainIssue.Id, + AssigneeId: domainIssue.AssigneeId, + AssigneeName: domainIssue.AssigneeName, + }) + } + return results, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} + +// loadSprintListIds returns the set of list ids in the folder that are sprint +// lists, so the task convertor can emit sprint_issues for their tasks. +func loadSprintListIds(db dal.Dal, connectionId uint64, folderId string) (map[string]bool, errors.Error) { + var lists []models.ClickUpList + if err := db.All(&lists, + dal.Select("list_id"), + dal.From(&models.ClickUpList{}), + dal.Where("connection_id = ? AND folder_id = ? AND is_sprint = ?", connectionId, folderId, true), + ); err != nil { + return nil, err + } + ids := make(map[string]bool, len(lists)) + for _, l := range lists { + ids[l.ListId] = true + } + return ids, nil +} + +// loadListTypeOverrides compiles the scope-config's Bug/IncidentListPattern and +// returns a map of list id -> forced issue type (INCIDENT beats BUG when a list +// matches both). Returns an empty map when neither pattern is set. +func loadListTypeOverrides(db dal.Dal, connectionId uint64, folderId string, sc *models.ClickUpScopeConfig) (map[string]string, errors.Error) { + if sc == nil || (sc.BugListPattern == "" && sc.IncidentListPattern == "") { + return map[string]string{}, nil + } + var bugRe, incidentRe *regexp.Regexp + if sc.BugListPattern != "" { + re, e := regexp.Compile(sc.BugListPattern) + if e != nil { + return nil, errors.Convert(e) + } + bugRe = re + } + if sc.IncidentListPattern != "" { + re, e := regexp.Compile(sc.IncidentListPattern) + if e != nil { + return nil, errors.Convert(e) + } + incidentRe = re + } + var lists []models.ClickUpList + if err := db.All(&lists, + dal.Select("list_id, name"), + dal.From(&models.ClickUpList{}), + dal.Where("connection_id = ? AND folder_id = ?", connectionId, folderId), + ); err != nil { + return nil, err + } + out := make(map[string]string) + for _, l := range lists { + if t := listTypeFor(l.Name, bugRe, incidentRe); t != "" { + out[l.ListId] = t + } + } + return out, nil +} + +// listTypeFor returns the forced issue type for a list name: INCIDENT when it +// matches incidentRe, else BUG when it matches bugRe, else "" (no override). +// INCIDENT is checked first so it wins when a name matches both. +func listTypeFor(name string, bugRe, incidentRe *regexp.Regexp) string { + switch { + case incidentRe != nil && incidentRe.MatchString(name): + return ticket.INCIDENT + case bugRe != nil && bugRe.MatchString(name): + return ticket.BUG + default: + return "" + } +} diff --git a/backend/plugins/clickup/tasks/task_convertor_test.go b/backend/plugins/clickup/tasks/task_convertor_test.go new file mode 100644 index 00000000000..8bb7b09273f --- /dev/null +++ b/backend/plugins/clickup/tasks/task_convertor_test.go @@ -0,0 +1,55 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "regexp" + "testing" + + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" +) + +func TestListTypeFor(t *testing.T) { + bug := regexp.MustCompile(`(?i)bug`) + incident := regexp.MustCompile(`(?i)incident`) + + tests := []struct { + name string + listName string + bugRe *regexp.Regexp + incRe *regexp.Regexp + want string + }{ + {"qa bugs list -> BUG", "QA Bugs", bug, incident, ticket.BUG}, + {"case-insensitive bug", "bug tracking", bug, incident, ticket.BUG}, + {"incident list -> INCIDENT", "Incident Backlog", bug, incident, ticket.INCIDENT}, + {"incident beats bug when both match", "Bug/Incident triage", bug, incident, ticket.INCIDENT}, + {"backlog matches neither -> empty", "Backlog", bug, incident, ""}, + {"sprint matches neither -> empty", "v4.3.0 Sprint 40", bug, incident, ""}, + {"nil bug regex, no incident -> empty", "QA Bugs", nil, nil, ""}, + {"only incident regex set", "Incident Backlog", nil, incident, ticket.INCIDENT}, + {"only bug regex set, name is incident -> empty", "Incident Backlog", bug, nil, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := listTypeFor(tt.listName, tt.bugRe, tt.incRe); got != tt.want { + t.Fatalf("listTypeFor(%q) = %q, want %q", tt.listName, got, tt.want) + } + }) + } +} diff --git a/backend/plugins/clickup/tasks/task_data.go b/backend/plugins/clickup/tasks/task_data.go new file mode 100644 index 00000000000..698e42b7220 --- /dev/null +++ b/backend/plugins/clickup/tasks/task_data.go @@ -0,0 +1,46 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "time" + + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +// ClickUpOptions are the per-scope options passed to a pipeline task. +type ClickUpOptions struct { + ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId,omitempty"` + FolderId string `json:"folderId" mapstructure:"folderId,omitempty"` + ScopeConfigId uint64 `json:"scopeConfigId" mapstructure:"scopeConfigId,omitempty"` + // TimeAfter limits collection to data created/updated after this time. + TimeAfter string `json:"timeAfter" mapstructure:"timeAfter,omitempty"` +} + +// ClickUpTaskData is the shared context handed to every ClickUp subtask. +type ClickUpTaskData struct { + Options *ClickUpOptions + ApiClient *api.ApiAsyncClient + TimeAfter *time.Time + // ScopeConfig carries the resolved scope config (status + type mapping). + // Never nil: PrepareTaskData defaults it to an empty config. + ScopeConfig *models.ClickUpScopeConfig +} + +type ClickUpApiParams models.ClickUpApiParams diff --git a/backend/plugins/clickup/tasks/task_extractor.go b/backend/plugins/clickup/tasks/task_extractor.go new file mode 100644 index 00000000000..f3d1209c702 --- /dev/null +++ b/backend/plugins/clickup/tasks/task_extractor.go @@ -0,0 +1,196 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "strconv" + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +var ExtractTaskMeta = plugin.SubTaskMeta{ + Name: "Extract Tasks", + EntryPoint: ExtractTasks, + EnabledByDefault: true, + Description: "Extract raw task data into the tool layer table _tool_clickup_tasks", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractTasks + +// ClickUpApiTask is the subset of the ClickUp task JSON that the extractor reads. +type ClickUpApiTask struct { + Id string `json:"id"` + CustomId string `json:"custom_id"` + Name string `json:"name"` + TextContent string `json:"text_content"` + Description string `json:"markdown_description"` + Status *struct { + Status string `json:"status"` + Type string `json:"type"` + } `json:"status"` + DateCreated string `json:"date_created"` + DateUpdated string `json:"date_updated"` + DateClosed string `json:"date_closed"` + Creator *struct { + Id json.Number `json:"id"` + Username string `json:"username"` + } `json:"creator"` + Assignees []struct { + Id json.Number `json:"id"` + Username string `json:"username"` + } `json:"assignees"` + Priority *struct { + Priority string `json:"priority"` + } `json:"priority"` + Parent string `json:"parent"` + Url string `json:"url"` + // Points is ClickUp's native sprint points field (Fibonacci LOE for these + // teams). It is the default story-point source. + Points *float64 `json:"points"` + CustomFields []clickUpCustomField `json:"custom_fields"` + TaskType string `json:"task_type"` + List *struct { + Id string `json:"id"` + } `json:"list"` + Space *struct { + Id string `json:"id"` + } `json:"space"` +} + +// clickUpCustomField is one entry of a task's custom_fields array. Value is left +// raw because ClickUp encodes it as a number or a string depending on the field. +type clickUpCustomField struct { + Name string `json:"name"` + Value json.RawMessage `json:"value"` +} + +func ExtractTasks(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*ClickUpTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: ClickUpApiParams{ + ConnectionId: data.Options.ConnectionId, + FolderId: data.Options.FolderId, + }, + Table: RAW_TASK_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiTask := &ClickUpApiTask{} + if err := errors.Convert(json.Unmarshal(row.Data, apiTask)); err != nil { + return nil, err + } + if apiTask.Id == "" { + return nil, nil + } + description := apiTask.Description + if description == "" { + description = apiTask.TextContent + } + listId := "" + if apiTask.List != nil { + listId = apiTask.List.Id + } + task := &models.ClickUpTask{ + ConnectionId: data.Options.ConnectionId, + Id: apiTask.Id, + ListId: listId, + FolderId: data.Options.FolderId, + CustomId: apiTask.CustomId, + Name: apiTask.Name, + Description: description, + Type: apiTask.TaskType, + ParentId: parentOf(apiTask.Parent), + StoryPoint: storyPointOf(apiTask, data.ScopeConfig), + Url: apiTask.Url, + CreatedDate: parseClickUpTime(apiTask.DateCreated), + UpdatedDate: parseClickUpTime(apiTask.DateUpdated), + ClosedDate: parseClickUpTime(apiTask.DateClosed), + } + if apiTask.Space != nil { + task.SpaceId = apiTask.Space.Id + } + if apiTask.Status != nil { + task.Status = apiTask.Status.Status + task.StatusType = apiTask.Status.Type + } + if apiTask.Priority != nil { + task.Priority = apiTask.Priority.Priority + } + if apiTask.Creator != nil { + task.CreatorId = apiTask.Creator.Id.String() + } + // TODO(clickup): ClickUp tasks can have multiple assignees. The MVP + // keeps only the first for Issue.AssigneeId; emitting one + // ticket.IssueAssignee row per assignee is a follow-up. + if len(apiTask.Assignees) > 0 { + task.AssigneeId = apiTask.Assignees[0].Id.String() + task.AssigneeName = apiTask.Assignees[0].Username + } + return []interface{}{task}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} + +// parentOf normalizes ClickUp's parent field, which is the JSON literal null +// (decoded to an empty string) for top-level tasks. +func parentOf(parent string) string { + if parent == "null" { + return "" + } + return parent +} + +// storyPointOf resolves a task's story points. Default source is ClickUp's +// native sprint points field. When the scope config names a custom field, that +// field's numeric value wins (teams that track Fibonacci LOE in a custom field). +func storyPointOf(apiTask *ClickUpApiTask, sc *models.ClickUpScopeConfig) *float64 { + if sc != nil && sc.StoryPointField != "" { + for _, cf := range apiTask.CustomFields { + if strings.EqualFold(cf.Name, sc.StoryPointField) { + return numericValue(cf.Value) + } + } + return nil + } + return apiTask.Points +} + +// numericValue coerces a raw custom-field value (JSON number or quoted string) +// into a float pointer; returns nil for empty / non-numeric values. +func numericValue(raw json.RawMessage) *float64 { + s := strings.TrimSpace(strings.Trim(string(raw), `"`)) + if s == "" || s == "null" { + return nil + } + f, err := strconv.ParseFloat(s, 64) + if err != nil { + return nil + } + return &f +} diff --git a/backend/plugins/clickup/tasks/user_collector.go b/backend/plugins/clickup/tasks/user_collector.go new file mode 100644 index 00000000000..47933cdc39d --- /dev/null +++ b/backend/plugins/clickup/tasks/user_collector.go @@ -0,0 +1,72 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const RAW_USER_TABLE = "clickup_users" + +// clickUpMemberListResponse mirrors the envelope returned by +// GET /list/{id}/member. +type clickUpMemberListResponse struct { + Members []json.RawMessage `json:"members"` +} + +var CollectUserMeta = plugin.SubTaskMeta{ + Name: "Collect Users", + EntryPoint: CollectUsers, + EnabledByDefault: true, + Description: "Collect the members of a ClickUp list", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, +} + +var _ plugin.SubTaskEntryPoint = CollectUsers + +func CollectUsers(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*ClickUpTaskData) + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: ClickUpApiParams{ + ConnectionId: data.Options.ConnectionId, + FolderId: data.Options.FolderId, + }, + Table: RAW_USER_TABLE, + }, + ApiClient: data.ApiClient, + UrlTemplate: "folder/{{ .Params.FolderId }}/member", + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var resp clickUpMemberListResponse + if err := api.UnmarshalResponse(res, &resp); err != nil { + return nil, err + } + return resp.Members, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/clickup/tasks/user_convertor.go b/backend/plugins/clickup/tasks/user_convertor.go new file mode 100644 index 00000000000..f8c91582b58 --- /dev/null +++ b/backend/plugins/clickup/tasks/user_convertor.go @@ -0,0 +1,88 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +var ConvertUserMeta = plugin.SubTaskMeta{ + Name: "Convert Users", + EntryPoint: ConvertUsers, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_clickup_users into domain layer table accounts", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + DependencyTables: []string{models.ClickUpUser{}.TableName()}, + ProductTables: []string{crossdomain.Account{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertUsers + +func ConvertUsers(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*ClickUpTaskData) + accountIdGen := didgen.NewDomainIdGenerator(&models.ClickUpUser{}) + + cursor, err := db.Cursor( + dal.From(&models.ClickUpUser{}), + dal.Where("connection_id = ?", data.Options.ConnectionId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: ClickUpApiParams{ + ConnectionId: data.Options.ConnectionId, + FolderId: data.Options.FolderId, + }, + Table: RAW_USER_TABLE, + }, + InputRowType: reflect.TypeOf(models.ClickUpUser{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + user := inputRow.(*models.ClickUpUser) + domainAccount := &crossdomain.Account{ + DomainEntity: domainlayer.DomainEntity{ + Id: accountIdGen.Generate(data.Options.ConnectionId, user.Id), + }, + UserName: user.Username, + FullName: user.Username, + Email: user.Email, + AvatarUrl: user.ProfilePicture, + } + return []interface{}{domainAccount}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/clickup/tasks/user_extractor.go b/backend/plugins/clickup/tasks/user_extractor.go new file mode 100644 index 00000000000..12f66c5639d --- /dev/null +++ b/backend/plugins/clickup/tasks/user_extractor.go @@ -0,0 +1,83 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +var ExtractUserMeta = plugin.SubTaskMeta{ + Name: "Extract Users", + EntryPoint: ExtractUsers, + EnabledByDefault: true, + Description: "Extract raw member data into the tool layer table _tool_clickup_users", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, +} + +var _ plugin.SubTaskEntryPoint = ExtractUsers + +// ClickUpApiUser is the subset of a ClickUp member JSON that the extractor reads. +type ClickUpApiUser struct { + Id json.Number `json:"id"` + Username string `json:"username"` + Email string `json:"email"` + Color string `json:"color"` + ProfilePicture string `json:"profilePicture"` +} + +func ExtractUsers(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*ClickUpTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: ClickUpApiParams{ + ConnectionId: data.Options.ConnectionId, + FolderId: data.Options.FolderId, + }, + Table: RAW_USER_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiUser := &ClickUpApiUser{} + if err := errors.Convert(json.Unmarshal(row.Data, apiUser)); err != nil { + return nil, err + } + id := apiUser.Id.String() + if id == "" { + return nil, nil + } + user := &models.ClickUpUser{ + ConnectionId: data.Options.ConnectionId, + Id: id, + Username: apiUser.Username, + Email: apiUser.Email, + Color: apiUser.Color, + ProfilePicture: apiUser.ProfilePicture, + } + return []interface{}{user}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/customize/tasks/customized_fields_extractor.go b/backend/plugins/customize/tasks/customized_fields_extractor.go index 7bd1d932d7b..9a8454c66b2 100644 --- a/backend/plugins/customize/tasks/customized_fields_extractor.go +++ b/backend/plugins/customize/tasks/customized_fields_extractor.go @@ -46,20 +46,28 @@ func ExtractCustomizedFields(taskCtx plugin.SubTaskContext) errors.Error { return nil } d := taskCtx.GetDal() - var err error + logger := taskCtx.GetLogger() for _, rule := range data.Options.TransformationRules { - err = extractCustomizedFields(taskCtx.GetContext(), d, rule.Table, rule.RawDataTable, rule.RawDataParams, rule.Mapping) + orphaned, err := extractCustomizedFields(taskCtx.GetContext(), d, rule.Table, rule.RawDataTable, rule.RawDataParams, rule.Mapping) if err != nil { return errors.Default.Wrap(err, "error extracting customized fields") } + if orphaned > 0 { + logger.Warn(nil, + "skipped %d row(s) of table %s: the raw record referenced by _raw_data_id no longer exists in %s", + orphaned, rule.Table, rule.RawDataTable) + } } return nil } -func extractCustomizedFields(ctx context.Context, d dal.Dal, table, rawTable, rawDataParams string, extractor map[string]string) error { +// extractCustomizedFields walks the domain layer table and copies configured JSON paths out of the +// raw record behind each row. It returns the number of rows skipped because no raw record backed +// them, so the caller can report that rather than leaving it invisible. +func extractCustomizedFields(ctx context.Context, d dal.Dal, table, rawTable, rawDataParams string, extractor map[string]string) (int, error) { pkFields, err := dal.GetPrimarykeyColumns(d, &models.Table{Name: table}) if err != nil { - return err + return 0, err } rawDataField := fmt.Sprintf("%s.data", rawTable) // `fields` only include `_raw_data_id` and primary keys coming from the domain layer table, and `data` coming from the raw layer @@ -76,21 +84,22 @@ func extractCustomizedFields(ctx context.Context, d dal.Dal, table, rawTable, ra } rows, err := d.Cursor(clauses...) if err != nil { - return err + return 0, err } defer rows.Close() + orphaned := 0 for rows.Next() { select { case <-ctx.Done(): - return ctx.Err() + return orphaned, ctx.Err() default: } row := make(map[string]interface{}) updates := make(map[string]interface{}) err = d.Fetch(rows, &row) if err != nil { - return err + return orphaned, err } switch blob := row["data"].(type) { case []byte: @@ -104,7 +113,7 @@ func extractCustomizedFields(ctx context.Context, d dal.Dal, table, rawTable, ra // special case for issues custom_fields rawDataId, ok := row["_raw_data_id"].(int64) if !ok { - return errors.Default.New("_raw_data_id is not int64") + return orphaned, errors.Default.New("_raw_data_id is not int64") } if table == "issues" && result.IsArray() { issueId := row["id"].(string) @@ -115,7 +124,7 @@ func extractCustomizedFields(ctx context.Context, d dal.Dal, table, rawTable, ra dal.Where("issue_id = ? AND field_id = ?", issueId, fieldId), ) if err != nil { - return err + return orphaned, err } result.ForEach(func(_, v gjson.Result) bool { @@ -142,7 +151,12 @@ func extractCustomizedFields(ctx context.Context, d dal.Dal, table, rawTable, ra } } default: - return nil + // The cursor LEFT JOINs the raw table, so a domain row whose _raw_data_id points at a + // raw record that has since been deleted comes back with a NULL data column. There is + // nothing to extract from it, but the rows after it are usually fine, so skip this one + // instead of ending the scan. + orphaned++ + continue } if len(updates) > 0 { @@ -151,15 +165,15 @@ func extractCustomizedFields(ctx context.Context, d dal.Dal, table, rawTable, ra delete(row, "data") query, params, err := mkUpdate(table, updates, row) if err != nil { - return err + return orphaned, err } err = d.Exec(query, params...) if err != nil { - return errors.Default.Wrap(err, "Exec SQL error") + return orphaned, errors.Default.Wrap(err, "Exec SQL error") } } } - return nil + return orphaned, rows.Err() } // fillInUpdates fills in the updates map with the result of the gjson query diff --git a/backend/plugins/customize/tasks/customized_fields_extractor_test.go b/backend/plugins/customize/tasks/customized_fields_extractor_test.go new file mode 100644 index 00000000000..9fc6ff894b0 --- /dev/null +++ b/backend/plugins/customize/tasks/customized_fields_extractor_test.go @@ -0,0 +1,150 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "context" + "testing" + + "github.com/apache/incubator-devlake/core/dal" + mockdal "github.com/apache/incubator-devlake/mocks/core/dal" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +// fetchedRow is one row the mocked cursor hands back, in the shape d.Fetch produces. +type fetchedRow map[string]interface{} + +// newExtractorMocks wires a Dal whose cursor yields the given rows in order. It returns the Dal +// mock plus a pointer to a slice collecting every UPDATE statement executed, so a test can tell +// which rows actually made it through. +func newExtractorMocks(t *testing.T, rows []fetchedRow) (*mockdal.Dal, *[]string) { + t.Helper() + + idColumn := new(mockdal.ColumnMeta) + idColumn.On("Name").Return("id").Maybe() + idColumn.On("PrimaryKey").Return(true, true).Maybe() + + cursor := new(mockdal.Rows) + // Next reports true once per row, then false to end the scan. + call := 0 + cursor.On("Next").Return(func() bool { + ok := call < len(rows) + call++ + return ok + }).Maybe() + cursor.On("Close").Return(nil).Maybe() + cursor.On("Err").Return(nil).Maybe() + + d := new(mockdal.Dal) + d.On("GetColumns", mock.Anything, mock.Anything). + Return([]dal.ColumnMeta{idColumn}, nil).Maybe() + d.On("Cursor", mock.Anything).Return(cursor, nil).Maybe() + + // Fetch copies the row for the current cursor position into the destination map. + fetched := 0 + d.On("Fetch", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + dst, ok := args.Get(1).(*map[string]interface{}) + if !ok { + t.Fatalf("Fetch called with %T, want *map[string]interface{}", args.Get(1)) + } + for k, v := range rows[fetched] { + (*dst)[k] = v + } + fetched++ + }).Return(nil).Maybe() + + executed := make([]string, 0) + d.On("Exec", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + executed = append(executed, args.Get(0).(string)) + }).Return(nil).Maybe() + + return d, &executed +} + +func Test_extractCustomizedFields_skipsOrphanedRowsAndKeepsScanning(t *testing.T) { + // The cursor LEFT JOINs the raw table, so a domain row whose raw record has been deleted comes + // back with a NULL data column. Before the fix that row ended the whole scan, leaving every + // later row unpopulated while the subtask still reported success. + rows := []fetchedRow{ + {"_raw_data_id": int64(1), "id": "ISSUE-1", "data": nil}, + {"_raw_data_id": int64(2), "id": "ISSUE-2", "data": `{"foo":"bar"}`}, + {"_raw_data_id": int64(3), "id": "ISSUE-3", "data": nil}, + {"_raw_data_id": int64(4), "id": "ISSUE-4", "data": `{"foo":"baz"}`}, + } + d, executed := newExtractorMocks(t, rows) + + orphaned, err := extractCustomizedFields( + context.Background(), d, "issues", "_raw_jira_api_issues", "params", + map[string]string{"x_custom": "foo"}) + + assert.NoError(t, err) + assert.Equal(t, 2, orphaned, "both rows without a raw record should be counted") + assert.Len(t, *executed, 2, "the two rows that do have raw data should still be updated") +} + +func Test_extractCustomizedFields_allRowsOrphanedIsNotAnError(t *testing.T) { + // A rule whose raw records have all been cleaned up is not a failure; it just has nothing to + // extract. The caller reports the count so it does not pass unnoticed. + rows := []fetchedRow{ + {"_raw_data_id": int64(1), "id": "ISSUE-1", "data": nil}, + {"_raw_data_id": int64(2), "id": "ISSUE-2", "data": nil}, + } + d, executed := newExtractorMocks(t, rows) + + orphaned, err := extractCustomizedFields( + context.Background(), d, "issues", "_raw_jira_api_issues", "params", + map[string]string{"x_custom": "foo"}) + + assert.NoError(t, err) + assert.Equal(t, 2, orphaned) + assert.Empty(t, *executed) +} + +func Test_extractCustomizedFields_noOrphansReportsZero(t *testing.T) { + rows := []fetchedRow{ + {"_raw_data_id": int64(1), "id": "ISSUE-1", "data": `{"foo":"bar"}`}, + {"_raw_data_id": int64(2), "id": "ISSUE-2", "data": `{"foo":"baz"}`}, + } + d, executed := newExtractorMocks(t, rows) + + orphaned, err := extractCustomizedFields( + context.Background(), d, "issues", "_raw_jira_api_issues", "params", + map[string]string{"x_custom": "foo"}) + + assert.NoError(t, err) + assert.Zero(t, orphaned) + assert.Len(t, *executed, 2) +} + +func Test_extractCustomizedFields_honoursCancelledContext(t *testing.T) { + rows := []fetchedRow{ + {"_raw_data_id": int64(1), "id": "ISSUE-1", "data": `{"foo":"bar"}`}, + } + d, executed := newExtractorMocks(t, rows) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := extractCustomizedFields( + ctx, d, "issues", "_raw_jira_api_issues", "params", + map[string]string{"x_custom": "foo"}) + + assert.ErrorIs(t, err, context.Canceled) + assert.Empty(t, *executed) +} diff --git a/backend/plugins/dbt/impl/impl.go b/backend/plugins/dbt/impl/impl.go index 5da8799fd87..c984187ca10 100644 --- a/backend/plugins/dbt/impl/impl.go +++ b/backend/plugins/dbt/impl/impl.go @@ -52,6 +52,7 @@ func (p Dbt) GetTablesInfo() []dal.Tabler { } func (p Dbt) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + taskCtx.GetLogger().Warn(nil, "The dbt plugin is deprecated and will be removed on August 31, 2026. Please migrate to alternative transformation approaches.") var op tasks.DbtOptions err := helper.Decode(options, &op, nil) if err != nil { diff --git a/backend/plugins/dora/e2e/change_lead_time/cicd_deployment_commits.csv b/backend/plugins/dora/e2e/change_lead_time/cicd_deployment_commits.csv index 9517c252010..9d1e488f072 100644 --- a/backend/plugins/dora/e2e/change_lead_time/cicd_deployment_commits.csv +++ b/backend/plugins/dora/e2e/change_lead_time/cicd_deployment_commits.csv @@ -14,4 +14,5 @@ id,result,started_date,duration_sec,cicd_deployment_id,cicd_scope_id,repo_url,en 13,SUCCESS,2023-04-13T07:56:39.000+00:00,60,pipeline7,cicd2,REPO111,PRODUCTION,3,,commit13,2023-4-13 7:56:39,2023-04-13T07:57:39.000+00:00 14,FAILURE,2023-04-13T07:57:26.000+00:00,60,pipeline8,cicd3,REPO111,PRODUCTION,,,commit14,2023-4-13 7:57:26,2023-04-13T07:58:26.000+00:00 15,SUCCESS,2023-04-13T07:57:45.000+00:00,60,pipeline9,cicd3,REPO111,PRODUCTION,,,commit15,2023-4-13 7:57:45,2023-04-13T07:58:45.000+00:00 -16,SUCCESS,2023-04-13T07:58:24.000+00:00,60,pipeline10,cicd3,REPO333,,,,commit16,2023-4-13 7:58:24,2023-04-13T07:59:24.000+00:00 \ No newline at end of file +16,SUCCESS,2023-04-13T07:58:24.000+00:00,60,pipeline10,cicd3,REPO333,,,,commit16,2023-4-13 7:58:24,2023-04-13T07:59:24.000+00:00 +17,SUCCESS,2023-04-13T07:59:00.000+00:00,60,pipeline11,cicd1,REPO111,PRODUCTION,,project1,direct_commit1,2023-4-13 7:59:00,2023-04-13T08:00:00.000+00:00 \ No newline at end of file diff --git a/backend/plugins/dora/e2e/change_lead_time/project_pr_metrics.csv b/backend/plugins/dora/e2e/change_lead_time/project_pr_metrics.csv index 1c1e045f5ec..1d9c736a503 100644 --- a/backend/plugins/dora/e2e/change_lead_time/project_pr_metrics.csv +++ b/backend/plugins/dora/e2e/change_lead_time/project_pr_metrics.csv @@ -5,3 +5,4 @@ pr2,project1,2537845559d8db99e9cda6190f32b50ec979c722,,comment04,1,60,5,1538,159 pr3,project1,55f445997abbd5918da59d202d28762cd56fbd44,5883,comment07,,5760,6,,10203,2023-04-07T04:51:47.000+00:00,2023-04-10T06:53:51.000+00:00,2023-04-11T06:53:51.000+00:00,2023-04-14T06:53:51.000+00:00,2023-04-13T07:30:34.000+00:00 pr4,project1,5ad0c09c447c19338f1dfbb65d89a3728962b3b7,11704,comment10,1500,,,,11764,2023-04-05T04:51:47.000+00:00,2023-04-14T08:55:01.000+00:00,2023-04-13T07:55:01.000+00:00,2023-04-13T08:55:01.000+00:00, pr5,project1,62535543802631a0d3daf0b0b78c6a7e05e508fb,13144,comment12,,313068,,,13204,2023-04-04T04:51:47.000+00:00,2022-09-07T23:07:13.000+00:00,2023-04-13T07:55:01.000+00:00,2023-04-13T08:55:01.000+00:00, +pr7,project1,pr7_commit0,1440,comment13,30,30,9,1433,2933,2023-04-11T07:00:00.000+00:00,2023-04-12T07:30:00.000+00:00,2023-04-12T07:00:00.000+00:00,2023-04-12T08:00:00.000+00:00,2023-04-13T07:52:26.000+00:00 \ No newline at end of file diff --git a/backend/plugins/dora/e2e/change_lead_time/pull_request_comments.csv b/backend/plugins/dora/e2e/change_lead_time/pull_request_comments.csv index 916a6f57d22..513e617b060 100644 --- a/backend/plugins/dora/e2e/change_lead_time/pull_request_comments.csv +++ b/backend/plugins/dora/e2e/change_lead_time/pull_request_comments.csv @@ -12,3 +12,4 @@ comment09,pr3,2023-4-12 6:53:51,i comment10,pr4,2023-4-14 8:55:01,j comment11,pr4,2023-4-14 8:55:01,k comment12,pr5,2022-09-07 23:07:13,l +comment13,pr7,2023-4-12 7:30:00,m \ No newline at end of file diff --git a/backend/plugins/dora/e2e/change_lead_time/pull_request_commits.csv b/backend/plugins/dora/e2e/change_lead_time/pull_request_commits.csv index a6e112d9fef..e183d14cce3 100644 --- a/backend/plugins/dora/e2e/change_lead_time/pull_request_commits.csv +++ b/backend/plugins/dora/e2e/change_lead_time/pull_request_commits.csv @@ -12,3 +12,4 @@ pr0_commit0,pr0,2022-1-10 4:51:47, 56b895f0443730c6d7abfbc51a05ab35abd2971f,pr4,2023-4-06 4:51:47, 5ad0c09c447c19338f1dfbb65d89a3728962b3b7,pr4,2023-4-05 4:51:47, 62535543802631a0d3daf0b0b78c6a7e05e508fb,pr5,2023-4-04 4:51:47, +pr7_commit0,pr7,2023-4-11 7:00:00, \ No newline at end of file diff --git a/backend/plugins/dora/e2e/change_lead_time/pull_requests.csv b/backend/plugins/dora/e2e/change_lead_time/pull_requests.csv index 46a2050777d..3a3ec7ae9af 100644 --- a/backend/plugins/dora/e2e/change_lead_time/pull_requests.csv +++ b/backend/plugins/dora/e2e/change_lead_time/pull_requests.csv @@ -6,3 +6,4 @@ pr3,repo1,a,pr_merge_commit3,2023-4-11 6:53:51,2023-4-14 6:53:51,deployment_comm pr4,repo1,,pr_merge_commit4,2023-4-13 7:55:01,2023-4-13 8:55:01,,, pr5,repo1,,pr_merge_commit5,2023-4-13 7:55:01,2023-4-13 8:55:01,,, pr6,repo1,,pr_merge_commit6,2023-4-13 7:55:01,,,, +pr7,repo1,a,commit9,2023-4-12 7:00:00,2023-4-12 8:00:00,,, diff --git a/backend/plugins/dora/tasks/change_lead_time_calculator.go b/backend/plugins/dora/tasks/change_lead_time_calculator.go index 47749b1af3f..9861a5c7a9e 100644 --- a/backend/plugins/dora/tasks/change_lead_time_calculator.go +++ b/backend/plugins/dora/tasks/change_lead_time_calculator.go @@ -277,47 +277,55 @@ func batchFetchFirstReviews(projectName string, db dal.Dal) (map[string]*code.Pu // batchFetchDeployments retrieves deployment commits for all merge commits in the given project. // Returns a map indexed by merge commit SHA for O(1) lookup performance. // -// The query finds the first successful production deployment for each merge commit by: -// 1. Finding deployment commits that have a previous successful deployment -// 2. Joining with commits_diffs to find which deployment included each merge commit -// 3. Filtering for successful production deployments -// 4. Ordering by started_date to get the earliest deployment +// Uses a two-phase strategy to avoid the "first deployment over-mapping" problem: // -// The map is indexed by merge_sha (from commits_diffs), not by deployment commit_sha, -// because the caller needs to look up deployments by PR merge_commit_sha. +// Phase 1 - Direct match: find successful PRODUCTION deployments whose commit_sha +// directly equals a PR's merge_commit_sha. Safe even for the very first deployment. +// +// Phase 2 - Diff-based fallback: use the commits_diffs join strategy, but deliberately +// skip the first deployment (prev_success_deployment_commit_id == "") to avoid over-mapping. func batchFetchDeployments(projectName string, db dal.Dal) (map[string]*devops.CicdDeploymentCommit, errors.Error) { - var results []*deploymentCommitWithMergeSha - - // Query finds the first deployment for each merge commit by using a window function - // to rank deployments by started_date, then filtering to keep only rank 1. + deploymentMap := make(map[string]*devops.CicdDeploymentCommit) + var directResults []*devops.CicdDeploymentCommit err := db.All( - &results, + &directResults, + dal.Select("dc.*"), + dal.From("cicd_deployment_commits dc"), + dal.Join("LEFT JOIN project_mapping pm ON pm.table = 'cicd_scopes' AND pm.row_id = dc.cicd_scope_id"), + dal.Where("dc.environment = 'PRODUCTION'"), // TODO: remove when multi-environment is supported + dal.Where("dc.result = ? AND pm.project_name = ?", devops.RESULT_SUCCESS, projectName), + dal.Orderby("dc.started_date ASC, dc.id ASC"), + ) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to batch fetch direct deployments") + } + for _, dc := range directResults { + if _, exists := deploymentMap[dc.CommitSha]; !exists { + deploymentCopy := *dc + deploymentMap[dc.CommitSha] = &deploymentCopy + } + } + var diffResults []*deploymentCommitWithMergeSha + err = db.All( + &diffResults, dal.Select("dc.*, cd.commit_sha as merge_sha"), dal.From("cicd_deployment_commits dc"), dal.Join("LEFT JOIN cicd_deployment_commits p ON dc.prev_success_deployment_commit_id = p.id"), dal.Join("INNER JOIN commits_diffs cd ON cd.new_commit_sha = dc.commit_sha AND cd.old_commit_sha = COALESCE(p.commit_sha, '')"), dal.Join("LEFT JOIN project_mapping pm ON pm.table = 'cicd_scopes' AND pm.row_id = dc.cicd_scope_id"), dal.Where("dc.prev_success_deployment_commit_id <> ''"), - dal.Where("dc.environment = 'PRODUCTION'"), // TODO: remove this when multi-environment is supported + dal.Where("dc.environment = 'PRODUCTION'"), // TODO: remove when multi-environment is supported dal.Where("dc.result = ? AND pm.project_name = ?", devops.RESULT_SUCCESS, projectName), dal.Orderby("cd.commit_sha, dc.started_date ASC, dc.id ASC"), ) - if err != nil { - return nil, errors.Default.Wrap(err, "failed to batch fetch deployments") + return nil, errors.Default.Wrap(err, "failed to batch fetch diff-based deployments") } - - // Build the map indexed by merge_sha for O(1) lookup. - // Keep only the first deployment for each merge commit (earliest by started_date). - deploymentMap := make(map[string]*devops.CicdDeploymentCommit, len(results)) - for _, result := range results { - // Only keep the first deployment for each merge_sha + for _, result := range diffResults { if _, exists := deploymentMap[result.MergeSha]; !exists { - // Copy the CicdDeploymentCommit without the MergeSha field deploymentCopy := result.CicdDeploymentCommit deploymentMap[result.MergeSha] = &deploymentCopy } } - return deploymentMap, nil } diff --git a/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_enterprise_ai_credit_usage.csv b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_enterprise_ai_credit_usage.csv new file mode 100644 index 00000000000..d9e0e72c229 --- /dev/null +++ b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_enterprise_ai_credit_usage.csv @@ -0,0 +1,4 @@ +connection_id,scope_id,year,month,day,enterprise,model,organization,user,product,cost_center_id,cost_center_name,gross_quantity,discount_quantity,net_quantity,price_per_unit,gross_amount,discount_amount,net_amount +1,octodemo,2025,12,10,octodemo,gpt-4.1,,,copilot,,,100.5,10.0,90.5,1.0,100.5,10.0,90.5 +1,octodemo,2025,12,10,octodemo,gpt-4o,,,copilot,cc-eng,Engineering,50.0,5.0,45.0,1.0,50.0,5.0,45.0 +1,octodemo,2025,12,10,octodemo,claude-3,,,copilot,cc-ml,Machine Learning,30.0,2.0,28.0,1.0,30.0,2.0,28.0 diff --git a/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_org_ai_credit_usage.csv b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_org_ai_credit_usage.csv new file mode 100644 index 00000000000..caf4f77cfed --- /dev/null +++ b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_org_ai_credit_usage.csv @@ -0,0 +1,4 @@ +connection_id,scope_id,year,month,day,organization,model,user,product,gross_quantity,discount_quantity,net_quantity,price_per_unit,gross_amount,discount_amount,net_amount +1,octodemo,2025,12,10,octodemo,gpt-4.1,alice,copilot,50.0,5.0,45.0,1.0,50.0,5.0,45.0 +1,octodemo,2025,12,10,octodemo,gpt-4o,bob,copilot,30.0,3.0,27.0,1.0,30.0,3.0,27.0 +1,octodemo,2025,12,10,octodemo,claude-3,charlie,copilot,20.0,0.0,20.0,1.0,20.0,0.0,20.0 diff --git a/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_user_ai_credit_usage.csv b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_user_ai_credit_usage.csv new file mode 100644 index 00000000000..7e098273481 --- /dev/null +++ b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_user_ai_credit_usage.csv @@ -0,0 +1,4 @@ +connection_id,scope_id,year,month,day,user,model,product,gross_quantity,discount_quantity,net_quantity,price_per_unit,gross_amount,discount_amount,net_amount +1,octodemo,2025,12,10,alice,gpt-4.1,copilot,50.0,5.0,45.0,1.0,50.0,5.0,45.0 +1,octodemo,2025,12,10,alice,gpt-4o,copilot,30.0,3.0,27.0,1.0,30.0,3.0,27.0 +1,octodemo,2025,12,10,bob,claude-3,copilot,20.0,2.0,18.0,1.0,20.0,2.0,18.0 diff --git a/backend/plugins/gh-copilot/models/enterprise_ai_credit_usage.go b/backend/plugins/gh-copilot/models/enterprise_ai_credit_usage.go new file mode 100644 index 00000000000..c10814000ce --- /dev/null +++ b/backend/plugins/gh-copilot/models/enterprise_ai_credit_usage.go @@ -0,0 +1,56 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// GhCopilotEnterpriseAiCreditUsage tracks AI credit consumption at the enterprise level. +// One row per time period per model per entity (user, org, or cost center). +type GhCopilotEnterpriseAiCreditUsage struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(191)" json:"scopeId"` + Year int `gorm:"primaryKey" json:"year"` + Month int `gorm:"primaryKey" json:"month"` + Day int `gorm:"primaryKey" json:"day"` + + Enterprise string `gorm:"primaryKey;type:varchar(191);comment:Enterprise slug" json:"enterprise"` + Model string `gorm:"primaryKey;type:varchar(191);comment:AI model name (e.g., gpt-4.1)" json:"model"` + Organization string `gorm:"index;type:varchar(255);comment:Organization within enterprise, if specified" json:"organization"` + User string `gorm:"index;type:varchar(255);comment:Username, if specified" json:"user"` + + Product string `gorm:"type:varchar(32);comment:Product name (e.g., copilot)" json:"product"` + CostCenterId string `gorm:"index;type:varchar(255);comment:Cost center identifier" json:"costCenterId"` + CostCenterName string `gorm:"type:varchar(255);comment:Cost center display name" json:"costCenterName"` + + // Credit usage breakdown + GrossQuantity float64 `json:"grossQuantity" gorm:"comment:Raw credits consumed"` + DiscountQuantity float64 `json:"discountQuantity" gorm:"comment:Credits discounted"` + NetQuantity float64 `json:"netQuantity" gorm:"comment:Credits after discount"` + PricePerUnit float64 `json:"pricePerUnit" gorm:"comment:Price per credit unit"` + GrossAmount float64 `json:"grossAmount" gorm:"comment:Gross cost before discount"` + DiscountAmount float64 `json:"discountAmount" gorm:"comment:Discount amount"` + NetAmount float64 `json:"netAmount" gorm:"comment:Net cost after discount"` + + common.NoPKModel +} + +func (GhCopilotEnterpriseAiCreditUsage) TableName() string { + return "_tool_copilot_enterprise_ai_credit_usage" +} diff --git a/backend/plugins/gh-copilot/models/migrationscripts/20260708_add_ai_credit_usage_metrics.go b/backend/plugins/gh-copilot/models/migrationscripts/20260708_add_ai_credit_usage_metrics.go new file mode 100644 index 00000000000..e968fb0dcfd --- /dev/null +++ b/backend/plugins/gh-copilot/models/migrationscripts/20260708_add_ai_credit_usage_metrics.go @@ -0,0 +1,162 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type addAiCreditUsageMetrics struct{} + +// --- Snapshot structs for migration (avoid importing models package to prevent drift) --- + +type creditUsageBreakdown20260708 struct { + GrossQuantity float64 + DiscountQuantity float64 + NetQuantity float64 + PricePerUnit float64 + GrossAmount float64 + DiscountAmount float64 + NetAmount float64 +} + +type enterpriseAiCreditUsage20260708 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(191)"` + Year int `gorm:"primaryKey"` + Month int `gorm:"primaryKey"` + Day int `gorm:"primaryKey"` + + Enterprise string `gorm:"primaryKey;type:varchar(191)"` + Model string `gorm:"primaryKey;type:varchar(191)"` + Organization string `gorm:"index;type:varchar(255)"` + User string `gorm:"index;type:varchar(255)"` + + Product string `gorm:"type:varchar(32)"` + CostCenterId string `gorm:"index;type:varchar(255)"` + CostCenterName string `gorm:"type:varchar(255)"` + + creditUsageBreakdown20260708 `gorm:"embedded"` + archived.NoPKModel +} + +func (enterpriseAiCreditUsage20260708) TableName() string { + return "_tool_copilot_enterprise_ai_credit_usage" +} + +type orgAiCreditUsage20260708 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(191)"` + Year int `gorm:"primaryKey"` + Month int `gorm:"primaryKey"` + Day int `gorm:"primaryKey"` + + Organization string `gorm:"primaryKey;type:varchar(191)"` + Model string `gorm:"primaryKey;type:varchar(191)"` + User string `gorm:"index;type:varchar(255)"` + + Product string `gorm:"type:varchar(32)"` + + creditUsageBreakdown20260708 `gorm:"embedded"` + archived.NoPKModel +} + +func (orgAiCreditUsage20260708) TableName() string { + return "_tool_copilot_org_ai_credit_usage" +} + +type userAiCreditUsage20260708 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(191)"` + Year int `gorm:"primaryKey"` + Month int `gorm:"primaryKey"` + Day int `gorm:"primaryKey"` + + User string `gorm:"primaryKey;type:varchar(191)"` + Model string `gorm:"primaryKey;type:varchar(191)"` + + Product string `gorm:"type:varchar(32)"` + + creditUsageBreakdown20260708 `gorm:"embedded"` + archived.NoPKModel +} + +func (userAiCreditUsage20260708) TableName() string { + return "_tool_copilot_user_ai_credit_usage" +} + +// userDailyMetrics20260708 adds AI credit, CLI and code-review columns to the existing table. +type userDailyMetrics20260708 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + UserId int64 `gorm:"primaryKey"` + + OrganizationId string `gorm:"type:varchar(100)"` + EnterpriseId string `gorm:"type:varchar(100)"` + UserLogin string `gorm:"type:varchar(255);index"` + UsedAgent bool + UsedChat bool + UsedCli bool `gorm:"comment:Whether user used Copilot CLI"` + UsedCopilotCodeReviewActive bool `gorm:"comment:Whether user actively used code review"` + UsedCopilotCodeReviewPassive bool `gorm:"comment:Whether user passively used code review"` + AiCreditsUsed float64 `gorm:"comment:AI credits consumed on this day"` + + UserInitiatedInteractionCount int + CodeGenerationActivityCount int + CodeAcceptanceActivityCount int + LocSuggestedToAddSum int + LocSuggestedToDeleteSum int + LocAddedSum int + LocDeletedSum int + + CliSessionCount int + CliRequestCount int + CliPromptCount int + CliOutputTokenSum int + CliPromptTokenSum int + + archived.NoPKModel +} + +func (userDailyMetrics20260708) TableName() string { + return "_tool_copilot_user_daily_metrics" +} + +func (u *addAiCreditUsageMetrics) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &enterpriseAiCreditUsage20260708{}, + &orgAiCreditUsage20260708{}, + &userAiCreditUsage20260708{}, + &userDailyMetrics20260708{}, + ) +} + +func (u *addAiCreditUsageMetrics) Version() uint64 { + return 20260708000000 +} + +func (u *addAiCreditUsageMetrics) Name() string { + return "add AI credit usage billing tables" +} diff --git a/backend/plugins/gh-copilot/models/migrationscripts/20260731_fix_ai_credit_usage_breakdown_columns.go b/backend/plugins/gh-copilot/models/migrationscripts/20260731_fix_ai_credit_usage_breakdown_columns.go new file mode 100644 index 00000000000..3365e9653de --- /dev/null +++ b/backend/plugins/gh-copilot/models/migrationscripts/20260731_fix_ai_credit_usage_breakdown_columns.go @@ -0,0 +1,98 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +// The 20260708 migration embedded the credit-usage breakdown columns through an +// *unexported* anonymous struct (`creditUsageBreakdown20260708`). GORM's schema +// parser does not migrate the fields of an unexported embedded type, so the +// gross_/discount_/net_ quantity and amount columns (plus price_per_unit) were +// never created even though the runtime models declare them inline. This +// follow-up migration adds the missing columns. AutoMigrate only adds columns +// that do not yet exist, so it is a no-op on databases that somehow already +// have them. + +type enterpriseAiCreditUsageBreakdown20260731 struct { + GrossQuantity float64 `gorm:"comment:Raw credits consumed"` + DiscountQuantity float64 `gorm:"comment:Credits discounted"` + NetQuantity float64 `gorm:"comment:Credits after discount"` + PricePerUnit float64 `gorm:"comment:Price per credit unit"` + GrossAmount float64 `gorm:"comment:Gross cost before discount"` + DiscountAmount float64 `gorm:"comment:Discount amount"` + NetAmount float64 `gorm:"comment:Net cost after discount"` + archived.NoPKModel +} + +func (enterpriseAiCreditUsageBreakdown20260731) TableName() string { + return "_tool_copilot_enterprise_ai_credit_usage" +} + +type orgAiCreditUsageBreakdown20260731 struct { + GrossQuantity float64 `gorm:"comment:Raw credits consumed"` + DiscountQuantity float64 `gorm:"comment:Credits discounted"` + NetQuantity float64 `gorm:"comment:Credits after discount"` + PricePerUnit float64 `gorm:"comment:Price per credit unit"` + GrossAmount float64 `gorm:"comment:Gross cost before discount"` + DiscountAmount float64 `gorm:"comment:Discount amount"` + NetAmount float64 `gorm:"comment:Net cost after discount"` + archived.NoPKModel +} + +func (orgAiCreditUsageBreakdown20260731) TableName() string { + return "_tool_copilot_org_ai_credit_usage" +} + +type userAiCreditUsageBreakdown20260731 struct { + GrossQuantity float64 `gorm:"comment:Raw credits consumed"` + DiscountQuantity float64 `gorm:"comment:Credits discounted"` + NetQuantity float64 `gorm:"comment:Credits after discount"` + PricePerUnit float64 `gorm:"comment:Price per credit unit"` + GrossAmount float64 `gorm:"comment:Gross cost before discount"` + DiscountAmount float64 `gorm:"comment:Discount amount"` + NetAmount float64 `gorm:"comment:Net cost after discount"` + archived.NoPKModel +} + +func (userAiCreditUsageBreakdown20260731) TableName() string { + return "_tool_copilot_user_ai_credit_usage" +} + +type fixAiCreditUsageBreakdownColumns struct{} + +func (u *fixAiCreditUsageBreakdownColumns) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &enterpriseAiCreditUsageBreakdown20260731{}, + &orgAiCreditUsageBreakdown20260731{}, + &userAiCreditUsageBreakdown20260731{}, + ) +} + +func (u *fixAiCreditUsageBreakdownColumns) Version() uint64 { + return 20260731000000 +} + +func (u *fixAiCreditUsageBreakdownColumns) Name() string { + return "add missing AI credit usage breakdown columns" +} diff --git a/backend/plugins/gh-copilot/models/migrationscripts/register.go b/backend/plugins/gh-copilot/models/migrationscripts/register.go index 399735695e0..8f190264618 100644 --- a/backend/plugins/gh-copilot/models/migrationscripts/register.go +++ b/backend/plugins/gh-copilot/models/migrationscripts/register.go @@ -31,5 +31,7 @@ func All() []plugin.MigrationScript { new(addPRFieldsToEnterpriseMetrics), new(addOrganizationIdToUserMetrics), new(addCopilotMetricsGaps), + new(addAiCreditUsageMetrics), + new(fixAiCreditUsageBreakdownColumns), } } diff --git a/backend/plugins/gh-copilot/models/models.go b/backend/plugins/gh-copilot/models/models.go index 5143ce5f8b7..814e45f4a51 100644 --- a/backend/plugins/gh-copilot/models/models.go +++ b/backend/plugins/gh-copilot/models/models.go @@ -47,5 +47,9 @@ func GetTablesInfo() []dal.Tabler { &GhCopilotSeat{}, // User-team mappings &GhCopilotUserTeam{}, + // AI credit usage billing (enterprise, org, user levels) + &GhCopilotEnterpriseAiCreditUsage{}, + &GhCopilotOrgAiCreditUsage{}, + &GhCopilotUserAiCreditUsage{}, } } diff --git a/backend/plugins/gh-copilot/models/models_test.go b/backend/plugins/gh-copilot/models/models_test.go index ef5b3eff6f5..7e9a2f2e979 100644 --- a/backend/plugins/gh-copilot/models/models_test.go +++ b/backend/plugins/gh-copilot/models/models_test.go @@ -41,6 +41,9 @@ func TestGetTablesInfo(t *testing.T) { (&GhCopilotUserMetricsByModelFeature{}).TableName(): false, (&GhCopilotSeat{}).TableName(): false, (&GhCopilotUserTeam{}).TableName(): false, + (&GhCopilotEnterpriseAiCreditUsage{}).TableName(): false, + (&GhCopilotOrgAiCreditUsage{}).TableName(): false, + (&GhCopilotUserAiCreditUsage{}).TableName(): false, } if len(tables) != len(expected) { diff --git a/backend/plugins/gh-copilot/models/org_ai_credit_usage.go b/backend/plugins/gh-copilot/models/org_ai_credit_usage.go new file mode 100644 index 00000000000..2e079f88f4d --- /dev/null +++ b/backend/plugins/gh-copilot/models/org_ai_credit_usage.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// GhCopilotOrgAiCreditUsage tracks AI credit consumption at the organization level. +// One row per time period per model per user (within the organization). +type GhCopilotOrgAiCreditUsage struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(191)" json:"scopeId"` + Year int `gorm:"primaryKey" json:"year"` + Month int `gorm:"primaryKey" json:"month"` + Day int `gorm:"primaryKey" json:"day"` + + Organization string `gorm:"primaryKey;type:varchar(191);comment:Organization name" json:"organization"` + Model string `gorm:"primaryKey;type:varchar(191);comment:AI model name (e.g., gpt-4.1)" json:"model"` + User string `gorm:"index;type:varchar(255);comment:Username consuming the credits" json:"user"` + + Product string `gorm:"type:varchar(32);comment:Product name (e.g., copilot)" json:"product"` + + // Credit usage breakdown + GrossQuantity float64 `json:"grossQuantity" gorm:"comment:Raw credits consumed"` + DiscountQuantity float64 `json:"discountQuantity" gorm:"comment:Credits discounted"` + NetQuantity float64 `json:"netQuantity" gorm:"comment:Credits after discount"` + PricePerUnit float64 `json:"pricePerUnit" gorm:"comment:Price per credit unit"` + GrossAmount float64 `json:"grossAmount" gorm:"comment:Gross cost before discount"` + DiscountAmount float64 `json:"discountAmount" gorm:"comment:Discount amount"` + NetAmount float64 `json:"netAmount" gorm:"comment:Net cost after discount"` + + common.NoPKModel +} + +func (GhCopilotOrgAiCreditUsage) TableName() string { + return "_tool_copilot_org_ai_credit_usage" +} diff --git a/backend/plugins/gh-copilot/models/user_ai_credit_usage.go b/backend/plugins/gh-copilot/models/user_ai_credit_usage.go new file mode 100644 index 00000000000..cb1ef2a6089 --- /dev/null +++ b/backend/plugins/gh-copilot/models/user_ai_credit_usage.go @@ -0,0 +1,52 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// GhCopilotUserAiCreditUsage tracks AI credit consumption at the individual user level. +// One row per time period per model per authenticated user. +type GhCopilotUserAiCreditUsage struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(191)" json:"scopeId"` + Year int `gorm:"primaryKey" json:"year"` + Month int `gorm:"primaryKey" json:"month"` + Day int `gorm:"primaryKey" json:"day"` + + User string `gorm:"primaryKey;type:varchar(191);comment:GitHub username" json:"user"` + Model string `gorm:"primaryKey;type:varchar(191);comment:AI model name (e.g., gpt-4.1)" json:"model"` + + Product string `gorm:"type:varchar(32);comment:Product name (e.g., copilot)" json:"product"` + + // Credit usage breakdown + GrossQuantity float64 `json:"grossQuantity" gorm:"comment:Raw credits consumed"` + DiscountQuantity float64 `json:"discountQuantity" gorm:"comment:Credits discounted"` + NetQuantity float64 `json:"netQuantity" gorm:"comment:Credits after discount"` + PricePerUnit float64 `json:"pricePerUnit" gorm:"comment:Price per credit unit"` + GrossAmount float64 `json:"grossAmount" gorm:"comment:Gross cost before discount"` + DiscountAmount float64 `json:"discountAmount" gorm:"comment:Discount amount"` + NetAmount float64 `json:"netAmount" gorm:"comment:Net cost after discount"` + + common.NoPKModel +} + +func (GhCopilotUserAiCreditUsage) TableName() string { + return "_tool_copilot_user_ai_credit_usage" +} diff --git a/backend/plugins/gh-copilot/models/user_metrics.go b/backend/plugins/gh-copilot/models/user_metrics.go index 18e9134c226..0a16b3bce82 100644 --- a/backend/plugins/gh-copilot/models/user_metrics.go +++ b/backend/plugins/gh-copilot/models/user_metrics.go @@ -30,14 +30,15 @@ type GhCopilotUserDailyMetrics struct { Day time.Time `gorm:"primaryKey;type:date" json:"day"` UserId int64 `gorm:"primaryKey" json:"userId"` - OrganizationId string `json:"organizationId" gorm:"type:varchar(100)"` - EnterpriseId string `json:"enterpriseId" gorm:"type:varchar(100)"` - UserLogin string `json:"userLogin" gorm:"type:varchar(255);index"` - UsedAgent bool `json:"usedAgent"` - UsedChat bool `json:"usedChat"` - UsedCli bool `json:"usedCli" gorm:"comment:Whether user used Copilot CLI"` - UsedCopilotCodeReviewActive bool `json:"usedCopilotCodeReviewActive" gorm:"comment:Whether user actively used code review"` - UsedCopilotCodeReviewPassive bool `json:"usedCopilotCodeReviewPassive" gorm:"comment:Whether user passively used code review"` + OrganizationId string `json:"organizationId" gorm:"type:varchar(100)"` + EnterpriseId string `json:"enterpriseId" gorm:"type:varchar(100)"` + UserLogin string `json:"userLogin" gorm:"type:varchar(255);index"` + UsedAgent bool `json:"usedAgent"` + UsedChat bool `json:"usedChat"` + UsedCli bool `json:"usedCli" gorm:"comment:Whether user used Copilot CLI"` + UsedCopilotCodeReviewActive bool `json:"usedCopilotCodeReviewActive" gorm:"comment:Whether user actively used code review"` + UsedCopilotCodeReviewPassive bool `json:"usedCopilotCodeReviewPassive" gorm:"comment:Whether user passively used code review"` + AiCreditsUsed float64 `json:"aiCreditsUsed" gorm:"comment:AI credits consumed on this day"` CopilotActivityMetrics `mapstructure:",squash"` CopilotCliMetrics `mapstructure:",squash"` diff --git a/backend/plugins/gh-copilot/service/api_path.go b/backend/plugins/gh-copilot/service/api_path.go new file mode 100644 index 00000000000..ec593c94671 --- /dev/null +++ b/backend/plugins/gh-copilot/service/api_path.go @@ -0,0 +1,28 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "fmt" + "net/url" + "strings" +) + +func copilotAPIPath(namespace, slug, resource string) string { + return fmt.Sprintf("%s/%s/%s", namespace, url.PathEscape(strings.TrimSpace(slug)), strings.TrimPrefix(resource, "/")) +} diff --git a/backend/plugins/gh-copilot/service/api_path_test.go b/backend/plugins/gh-copilot/service/api_path_test.go new file mode 100644 index 00000000000..4a57cc1e44b --- /dev/null +++ b/backend/plugins/gh-copilot/service/api_path_test.go @@ -0,0 +1,30 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCopilotAPIPathPreservesHyphenatedEnterpriseSlug(t *testing.T) { + path := copilotAPIPath("enterprises", "my-enterprise", "copilot/billing/seats") + + assert.Equal(t, "enterprises/my-enterprise/copilot/billing/seats", path) +} diff --git a/backend/plugins/gh-copilot/service/connection_test_helper.go b/backend/plugins/gh-copilot/service/connection_test_helper.go index 9cda8ae1dd4..8891476020f 100644 --- a/backend/plugins/gh-copilot/service/connection_test_helper.go +++ b/backend/plugins/gh-copilot/service/connection_test_helper.go @@ -85,7 +85,7 @@ func TestConnection(ctx stdctx.Context, br corectx.BasicRes, connection *models. // Note: /enterprises/{ent}/copilot/billing does not exist — use /billing/seats instead. if hasEnterprise { entSlug := strings.TrimSpace(connection.Enterprise) - seatsPath := fmt.Sprintf("enterprises/%s/copilot/billing/seats", entSlug) + seatsPath := copilotAPIPath("enterprises", entSlug, "copilot/billing/seats") entSummary, entErr := fetchSeatsSummary(apiClient, seatsPath) if entErr != nil { return nil, entErr @@ -97,7 +97,7 @@ func TestConnection(ctx stdctx.Context, br corectx.BasicRes, connection *models. // Test org endpoint when configured. if hasOrg { - orgSummary, orgErr := fetchBillingSummary(apiClient, fmt.Sprintf("orgs/%s/copilot/billing", connection.Organization)) + orgSummary, orgErr := fetchBillingSummary(apiClient, copilotAPIPath("orgs", connection.Organization, "copilot/billing")) if orgErr != nil { return nil, orgErr } diff --git a/backend/plugins/gh-copilot/tasks/ai_credit_collector.go b/backend/plugins/gh-copilot/tasks/ai_credit_collector.go new file mode 100644 index 00000000000..f9dd4d75b3a --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/ai_credit_collector.go @@ -0,0 +1,123 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const rawAiCreditUsageTable = "copilot_ai_credit_usage" + +func CollectAiCreditUsage(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*GhCopilotTaskData) + if !ok { + return errors.Default.New("task data is not GhCopilotTaskData") + } + connection := data.Connection + connection.Normalize() + + apiClient, err := CreateApiClient(taskCtx.TaskContext(), connection) + if err != nil { + return err + } + + var urlTemplate string + var scope string + + if connection.HasEnterprise() { + urlTemplate = fmt.Sprintf("enterprises/%s/settings/billing/ai_credit/usage", connection.Enterprise) + scope = connection.Enterprise + } else if connection.Organization != "" { + urlTemplate = fmt.Sprintf("organizations/%s/settings/billing/ai_credit/usage", connection.Organization) + scope = connection.Organization + } else { + urlTemplate = "user/settings/billing/ai_credit/usage" + scope = "user" + } + + rawArgs := helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawAiCreditUsageTable, + Options: copilotRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: connection.Endpoint, + }, + } + + collector, err := helper.NewStatefulApiCollector(rawArgs) + if err != nil { + return err + } + + now := time.Now().UTC() + start, until := computeReportDateRange(now, collector.GetSince()) + start = clampDailyMetricsStartForBackfill(start, until) + + dayIter := newDayIterator(start, until) + + err = collector.InitCollector(helper.ApiCollectorArgs{ + ApiClient: apiClient, + Input: dayIter, + UrlTemplate: urlTemplate, + Query: func(reqData *helper.RequestData) (url.Values, errors.Error) { + input := reqData.Input.(*dayInput) + day, parseErr := time.Parse("2006-01-02", input.Day) + if parseErr != nil { + return nil, errors.Convert(parseErr) + } + q := url.Values{} + q.Set("year", strconv.Itoa(day.Year())) + q.Set("month", strconv.Itoa(int(day.Month()))) + q.Set("day", strconv.Itoa(day.Day())) + return q, nil + }, + Incremental: true, + Concurrency: 1, + AfterResponse: ignoreNoContent, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + if res.StatusCode != http.StatusOK { + return nil, errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("failed to collect AI credit usage for %s", scope)) + } + + var response struct { + UsageItems []json.RawMessage `json:"usageItems"` + } + if unmErr := helper.UnmarshalResponse(res, &response); unmErr != nil { + return nil, unmErr + } + + return response.UsageItems, nil + }, + }) + if err != nil { + return err + } + + return collector.Execute() +} diff --git a/backend/plugins/gh-copilot/tasks/ai_credit_extractor.go b/backend/plugins/gh-copilot/tasks/ai_credit_extractor.go new file mode 100644 index 00000000000..7d372db2629 --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/ai_credit_extractor.go @@ -0,0 +1,194 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/gh-copilot/models" +) + +// aiCreditUsageRecord represents a single usage item from the AI credit usage API. +type aiCreditUsageRecord struct { + Product string `json:"product"` + Sku string `json:"sku"` + Model string `json:"model"` + UnitType string `json:"unitType"` + PricePerUnit float64 `json:"pricePerUnit"` + GrossQuantity float64 `json:"grossQuantity"` + DiscountQuantity float64 `json:"discountQuantity"` + NetQuantity float64 `json:"netQuantity"` + GrossAmount float64 `json:"grossAmount"` + DiscountAmount float64 `json:"discountAmount"` + NetAmount float64 `json:"netAmount"` +} + +// aiCreditResponseWrapper represents the wrapper around the API response containing time period and usage items. +type aiCreditResponseWrapper struct { + TimePeriod struct { + Year int `json:"year"` + Month int `json:"month"` + Day int `json:"day"` + } `json:"timePeriod"` + Enterprise string `json:"enterprise"` + Organization string `json:"organization"` + User string `json:"user"` + Product string `json:"product"` + Model string `json:"model"` + CostCenter struct { + Id string `json:"id"` + Name string `json:"name"` + } `json:"costCenter"` +} + +// ExtractAiCreditUsage parses AI credit usage records into the appropriate model tables. +func ExtractAiCreditUsage(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*GhCopilotTaskData) + if !ok { + return errors.Default.New("task data is not GhCopilotTaskData") + } + connection := data.Connection + connection.Normalize() + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawAiCreditUsageTable, + Options: copilotRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: connection.Endpoint, + }, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + // Parse raw data + var record aiCreditUsageRecord + err := json.Unmarshal(row.Data, &record) + if err != nil { + return nil, errors.Convert(err) + } + + // Extract wrapper info from row context + var wrapper aiCreditResponseWrapper + if connection.HasEnterprise() { + wrapper.Enterprise = connection.Enterprise + } else if connection.Organization != "" { + wrapper.Organization = connection.Organization + } + wrapper.Product = record.Product + wrapper.Model = record.Model + + // Derive the time period from the collector's day input so records are + // deterministic and aligned with the requested billing day, rather than + // depending on the extraction-time clock. + var input dayInput + if len(row.Input) > 0 { + if unmErr := json.Unmarshal(row.Input, &input); unmErr != nil { + return nil, errors.Convert(unmErr) + } + } + day, parseErr := time.Parse("2006-01-02", input.Day) + if parseErr != nil { + return nil, errors.Convert(parseErr) + } + wrapper.TimePeriod.Year = day.Year() + wrapper.TimePeriod.Month = int(day.Month()) + wrapper.TimePeriod.Day = day.Day() + + var results []interface{} + + // Route to appropriate table based on connection type + if connection.HasEnterprise() { + toolRecord := &models.GhCopilotEnterpriseAiCreditUsage{ + ConnectionId: data.Connection.ID, + ScopeId: data.Options.ScopeId, + Year: wrapper.TimePeriod.Year, + Month: wrapper.TimePeriod.Month, + Day: wrapper.TimePeriod.Day, + Enterprise: wrapper.Enterprise, + Model: record.Model, + Organization: wrapper.Organization, + User: wrapper.User, + Product: record.Product, + CostCenterId: wrapper.CostCenter.Id, + CostCenterName: wrapper.CostCenter.Name, + GrossQuantity: record.GrossQuantity, + DiscountQuantity: record.DiscountQuantity, + NetQuantity: record.NetQuantity, + PricePerUnit: record.PricePerUnit, + GrossAmount: record.GrossAmount, + DiscountAmount: record.DiscountAmount, + NetAmount: record.NetAmount, + } + results = append(results, toolRecord) + } else if connection.Organization != "" { + toolRecord := &models.GhCopilotOrgAiCreditUsage{ + ConnectionId: data.Connection.ID, + ScopeId: data.Options.ScopeId, + Year: wrapper.TimePeriod.Year, + Month: wrapper.TimePeriod.Month, + Day: wrapper.TimePeriod.Day, + Organization: wrapper.Organization, + Model: record.Model, + User: wrapper.User, + Product: record.Product, + GrossQuantity: record.GrossQuantity, + DiscountQuantity: record.DiscountQuantity, + NetQuantity: record.NetQuantity, + PricePerUnit: record.PricePerUnit, + GrossAmount: record.GrossAmount, + DiscountAmount: record.DiscountAmount, + NetAmount: record.NetAmount, + } + results = append(results, toolRecord) + } else { + // User-level credits + toolRecord := &models.GhCopilotUserAiCreditUsage{ + ConnectionId: data.Connection.ID, + ScopeId: data.Options.ScopeId, + Year: wrapper.TimePeriod.Year, + Month: wrapper.TimePeriod.Month, + Day: wrapper.TimePeriod.Day, + User: connection.Name, // Use connection display name + Model: record.Model, + Product: record.Product, + GrossQuantity: record.GrossQuantity, + DiscountQuantity: record.DiscountQuantity, + NetQuantity: record.NetQuantity, + PricePerUnit: record.PricePerUnit, + GrossAmount: record.GrossAmount, + DiscountAmount: record.DiscountAmount, + NetAmount: record.NetAmount, + } + results = append(results, toolRecord) + } + + return results, nil + }, + }) + if err != nil { + return err + } + + return extractor.Execute() +} diff --git a/backend/plugins/gh-copilot/tasks/api_path.go b/backend/plugins/gh-copilot/tasks/api_path.go new file mode 100644 index 00000000000..0d177ee14be --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/api_path.go @@ -0,0 +1,28 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "net/url" + "strings" +) + +func copilotAPIPath(namespace, slug, resource string) string { + return fmt.Sprintf("%s/%s/%s", namespace, url.PathEscape(strings.TrimSpace(slug)), strings.TrimPrefix(resource, "/")) +} diff --git a/backend/plugins/gh-copilot/tasks/api_path_test.go b/backend/plugins/gh-copilot/tasks/api_path_test.go new file mode 100644 index 00000000000..b92420484f3 --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/api_path_test.go @@ -0,0 +1,30 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCopilotAPIPathPreservesHyphenatedEnterpriseSlug(t *testing.T) { + path := copilotAPIPath("enterprises", "my-enterprise", "copilot/billing/seats") + + require.Equal(t, "enterprises/my-enterprise/copilot/billing/seats", path) +} diff --git a/backend/plugins/gh-copilot/tasks/enterprise_metrics_collector.go b/backend/plugins/gh-copilot/tasks/enterprise_metrics_collector.go index 16061cf5096..e153d59c948 100644 --- a/backend/plugins/gh-copilot/tasks/enterprise_metrics_collector.go +++ b/backend/plugins/gh-copilot/tasks/enterprise_metrics_collector.go @@ -19,7 +19,6 @@ package tasks import ( "encoding/json" - "fmt" "net/http" "net/url" "time" @@ -81,10 +80,9 @@ func CollectEnterpriseMetrics(taskCtx plugin.SubTaskContext) errors.Error { dayIter := newDayIterator(start, until) err = collector.InitCollector(helper.ApiCollectorArgs{ - ApiClient: apiClient, - Input: dayIter, - UrlTemplate: fmt.Sprintf("enterprises/%s/copilot/metrics/reports/enterprise-1-day", - connection.Enterprise), + ApiClient: apiClient, + Input: dayIter, + UrlTemplate: copilotAPIPath("enterprises", connection.Enterprise, "copilot/metrics/reports/enterprise-1-day"), Query: func(reqData *helper.RequestData) (url.Values, errors.Error) { input := reqData.Input.(*dayInput) q := url.Values{} diff --git a/backend/plugins/gh-copilot/tasks/metrics_collector_test.go b/backend/plugins/gh-copilot/tasks/metrics_collector_test.go index d71d20d4440..88214af6db6 100644 --- a/backend/plugins/gh-copilot/tasks/metrics_collector_test.go +++ b/backend/plugins/gh-copilot/tasks/metrics_collector_test.go @@ -18,12 +18,11 @@ limitations under the License. package tasks import ( - "bytes" - "io" "net/http" "testing" "time" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/stretchr/testify/require" ) @@ -109,35 +108,20 @@ func TestUserMetricsDateRangeAppliesFourDayBackfillWindow(t *testing.T) { require.Equal(t, time.Date(2025, 1, 6, 0, 0, 0, 0, time.UTC), start) } -func TestParseReportMetadataResponseNoContent(t *testing.T) { - res := &http.Response{ - StatusCode: http.StatusNoContent, - Body: io.NopCloser(bytes.NewReader(nil)), - } - - meta, err := parseReportMetadataResponse(res, nil) - require.NoError(t, err) - require.Nil(t, meta) +func TestIgnoreNoContentSkips204And404(t *testing.T) { + require.Equal(t, helper.ErrIgnoreAndContinue, ignoreNoContent(&http.Response{StatusCode: http.StatusNoContent})) + require.Equal(t, helper.ErrIgnoreAndContinue, ignoreNoContent(&http.Response{StatusCode: http.StatusNotFound})) + require.NoError(t, ignoreNoContent(&http.Response{StatusCode: http.StatusOK})) } -func TestParseReportMetadataResponseEmptyBody(t *testing.T) { - res := &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewReader(nil)), - } - - meta, err := parseReportMetadataResponse(res, nil) +func TestParseReportMetadataEmptyBody(t *testing.T) { + meta, err := parseReportMetadata(nil, nil) require.NoError(t, err) require.Nil(t, meta) } -func TestParseReportMetadataResponseEmptyString(t *testing.T) { - res := &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewReader([]byte(`""`))), - } - - meta, err := parseReportMetadataResponse(res, nil) +func TestParseReportMetadataEmptyString(t *testing.T) { + meta, err := parseReportMetadata([]byte(`""`), nil) require.NoError(t, err) require.Nil(t, meta) } diff --git a/backend/plugins/gh-copilot/tasks/org_metrics_collector.go b/backend/plugins/gh-copilot/tasks/org_metrics_collector.go index c3a8b5e4409..c75ce7b6ce6 100644 --- a/backend/plugins/gh-copilot/tasks/org_metrics_collector.go +++ b/backend/plugins/gh-copilot/tasks/org_metrics_collector.go @@ -19,7 +19,6 @@ package tasks import ( "encoding/json" - "fmt" "io" "net/http" "net/url" @@ -76,10 +75,9 @@ func CollectOrgMetrics(taskCtx plugin.SubTaskContext) errors.Error { dayIter := newDayIterator(start, until) err = collector.InitCollector(helper.ApiCollectorArgs{ - ApiClient: apiClient, - Input: dayIter, - UrlTemplate: fmt.Sprintf("orgs/%s/copilot/metrics/reports/organization-1-day", - connection.Organization), + ApiClient: apiClient, + Input: dayIter, + UrlTemplate: copilotAPIPath("orgs", connection.Organization, "copilot/metrics/reports/organization-1-day"), Query: func(reqData *helper.RequestData) (url.Values, errors.Error) { input := reqData.Input.(*dayInput) q := url.Values{} diff --git a/backend/plugins/gh-copilot/tasks/register.go b/backend/plugins/gh-copilot/tasks/register.go index 3c7e5b1eeb9..23d592304e9 100644 --- a/backend/plugins/gh-copilot/tasks/register.go +++ b/backend/plugins/gh-copilot/tasks/register.go @@ -28,11 +28,13 @@ func GetSubTaskMetas() []plugin.SubTaskMeta { CollectEnterpriseMetricsMeta, CollectUserMetricsMeta, CollectUserTeamsMeta, + CollectAiCreditUsageMeta, // Extractors ExtractSeatsMeta, ExtractOrgMetricsMeta, ExtractEnterpriseMetricsMeta, ExtractUserMetricsMeta, ExtractUserTeamsMeta, + ExtractAiCreditUsageMeta, } } diff --git a/backend/plugins/gh-copilot/tasks/report_download_helper.go b/backend/plugins/gh-copilot/tasks/report_download_helper.go index 236b039889b..280e418e30c 100644 --- a/backend/plugins/gh-copilot/tasks/report_download_helper.go +++ b/backend/plugins/gh-copilot/tasks/report_download_helper.go @@ -95,15 +95,6 @@ type reportMetadataResponse struct { ReportEndDay string `json:"report_end_day"` } -func readReportMetadataBody(res *http.Response) ([]byte, errors.Error) { - body, readErr := io.ReadAll(res.Body) - res.Body.Close() - if readErr != nil { - return nil, errors.Default.Wrap(readErr, "failed to read report metadata") - } - return body, nil -} - func logReportMetadataParseError(body []byte, err error, logger log.Logger) { if logger == nil { return @@ -165,23 +156,6 @@ func parseReportMetadata(body []byte, logger log.Logger) (*reportMetadataRespons return &meta, nil } -func parseReportMetadataResponse(res *http.Response, logger log.Logger) (*reportMetadataResponse, errors.Error) { - if res.StatusCode == http.StatusNoContent { - if logger != nil { - logger.Info("Report metadata not ready yet (204), skipping for now") - } - res.Body.Close() - return nil, nil - } - - body, readErr := readReportMetadataBody(res) - if readErr != nil { - return nil, readErr - } - - return parseReportMetadata(body, logger) -} - func collectRawReportRecords(meta *reportMetadataResponse, logger log.Logger) ([]json.RawMessage, errors.Error) { if len(meta.DownloadLinks) == 0 { logger.Info("No download links for report day=%s, skipping", meta.ReportDay) @@ -212,17 +186,11 @@ func parseRawReportResponse(res *http.Response, logger log.Logger) ([]json.RawMe return nil, nil } - var meta *reportMetadataResponse - if jsonErr := json.Unmarshal(body, &meta); jsonErr != nil { - snippet := string(body) - if len(snippet) > 200 { - snippet = snippet[:200] - } - logger.Error(jsonErr, "failed to parse report metadata, body=%s", snippet) - return nil, errors.Default.Wrap(jsonErr, "failed to parse report metadata") - } - - meta, err := parseReportMetadataResponse(res, logger) + // Parse the metadata from the body we already read above. Previously this + // re-read res.Body via parseReportMetadataResponse, but the body had already + // been consumed by io.ReadAll, so the second read returned empty and the + // collector silently produced zero records (affecting enterprise metrics). + meta, err := parseReportMetadata(body, logger) if err != nil || meta == nil { return nil, err } diff --git a/backend/plugins/gh-copilot/tasks/seat_collector.go b/backend/plugins/gh-copilot/tasks/seat_collector.go index 8724805f666..546a0e30abc 100644 --- a/backend/plugins/gh-copilot/tasks/seat_collector.go +++ b/backend/plugins/gh-copilot/tasks/seat_collector.go @@ -90,9 +90,9 @@ func CollectCopilotSeatAssignments(taskCtx plugin.SubTaskContext) errors.Error { var urlTemplate string switch { case connection.HasEnterprise(): - urlTemplate = fmt.Sprintf("enterprises/%s/copilot/billing/seats", connection.Enterprise) + urlTemplate = copilotAPIPath("enterprises", connection.Enterprise, "copilot/billing/seats") case connection.Organization != "": - urlTemplate = fmt.Sprintf("orgs/%s/copilot/billing/seats", connection.Organization) + urlTemplate = copilotAPIPath("orgs", connection.Organization, "copilot/billing/seats") default: taskCtx.GetLogger().Warn(nil, "skipping seat collection: no enterprise or organization configured on connection %d", connection.ID) return nil diff --git a/backend/plugins/gh-copilot/tasks/subtasks.go b/backend/plugins/gh-copilot/tasks/subtasks.go index 61ed5799525..69255287026 100644 --- a/backend/plugins/gh-copilot/tasks/subtasks.go +++ b/backend/plugins/gh-copilot/tasks/subtasks.go @@ -105,3 +105,20 @@ var ExtractUserTeamsMeta = plugin.SubTaskMeta{ Description: "Extract Copilot user-team mappings into tool-layer table", Dependencies: []*plugin.SubTaskMeta{&CollectUserTeamsMeta}, } + +var CollectAiCreditUsageMeta = plugin.SubTaskMeta{ + Name: "collectAiCreditUsage", + EntryPoint: CollectAiCreditUsage, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Collect GitHub Copilot AI credit usage billing data from billing API", +} + +var ExtractAiCreditUsageMeta = plugin.SubTaskMeta{ + Name: "extractAiCreditUsage", + EntryPoint: ExtractAiCreditUsage, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Extract Copilot AI credit usage into tool-layer billing tables", + Dependencies: []*plugin.SubTaskMeta{&CollectAiCreditUsageMeta}, +} diff --git a/backend/plugins/gh-copilot/tasks/user_metrics_collector.go b/backend/plugins/gh-copilot/tasks/user_metrics_collector.go index a092a0450b8..d24db69f411 100644 --- a/backend/plugins/gh-copilot/tasks/user_metrics_collector.go +++ b/backend/plugins/gh-copilot/tasks/user_metrics_collector.go @@ -19,7 +19,6 @@ package tasks import ( "encoding/json" - "fmt" "io" "net/http" "net/url" @@ -63,12 +62,11 @@ func parseUserMetricsReportResponse(res *http.Response, logger log.Logger) ([]js return nil, nil } - var meta *reportMetadataResponse - if jsonErr := json.Unmarshal(body, &meta); jsonErr != nil { - return nil, errors.Default.Wrap(jsonErr, "failed to parse report metadata") - } - - meta, err := parseReportMetadataResponse(res, logger) + // Parse the metadata from the body we already read above. Previously this + // re-read res.Body via parseReportMetadataResponse, but the body had already + // been consumed by io.ReadAll, so the second read returned empty and the + // collector silently produced zero user-metrics records. + meta, err := parseReportMetadata(body, logger) if err != nil || meta == nil { return nil, err } @@ -95,9 +93,9 @@ func CollectUserMetrics(taskCtx plugin.SubTaskContext) errors.Error { var urlTemplate string if connection.HasEnterprise() { - urlTemplate = fmt.Sprintf("enterprises/%s/copilot/metrics/reports/users-1-day", connection.Enterprise) + urlTemplate = copilotAPIPath("enterprises", connection.Enterprise, "copilot/metrics/reports/users-1-day") } else if connection.Organization != "" { - urlTemplate = fmt.Sprintf("orgs/%s/copilot/metrics/reports/users-1-day", connection.Organization) + urlTemplate = copilotAPIPath("orgs", connection.Organization, "copilot/metrics/reports/users-1-day") } else { return nil } diff --git a/backend/plugins/gh-copilot/tasks/user_metrics_extractor.go b/backend/plugins/gh-copilot/tasks/user_metrics_extractor.go index 72992194063..7671fab86a9 100644 --- a/backend/plugins/gh-copilot/tasks/user_metrics_extractor.go +++ b/backend/plugins/gh-copilot/tasks/user_metrics_extractor.go @@ -49,6 +49,7 @@ type userDailyReport struct { UsedCli bool `json:"used_cli"` UsedCopilotCodeReviewActive bool `json:"used_copilot_code_review_active"` UsedCopilotCodeReviewPassive bool `json:"used_copilot_code_review_passive"` + AiCreditsUsed float64 `json:"ai_credits_used"` TotalsByIde []userTotalsByIde `json:"totals_by_ide"` TotalsByFeature []totalsByFeature `json:"totals_by_feature"` TotalsByLanguageFeature []totalsByLangFeature `json:"totals_by_language_feature"` @@ -123,6 +124,7 @@ func ExtractUserMetrics(taskCtx plugin.SubTaskContext) errors.Error { UsedCli: u.UsedCli, UsedCopilotCodeReviewActive: u.UsedCopilotCodeReviewActive, UsedCopilotCodeReviewPassive: u.UsedCopilotCodeReviewPassive, + AiCreditsUsed: u.AiCreditsUsed, CopilotActivityMetrics: models.CopilotActivityMetrics{ UserInitiatedInteractionCount: u.UserInitiatedInteractionCount, CodeGenerationActivityCount: u.CodeGenerationActivityCount, diff --git a/backend/plugins/gh-copilot/tasks/user_teams_collector.go b/backend/plugins/gh-copilot/tasks/user_teams_collector.go index 2ae0200d2ef..dc3287e5c5b 100644 --- a/backend/plugins/gh-copilot/tasks/user_teams_collector.go +++ b/backend/plugins/gh-copilot/tasks/user_teams_collector.go @@ -19,7 +19,6 @@ package tasks import ( "encoding/json" - "fmt" "io" "net/http" "net/url" @@ -45,9 +44,9 @@ func CollectUserTeams(taskCtx plugin.SubTaskContext) errors.Error { var urlTemplate string if connection.HasEnterprise() { - urlTemplate = fmt.Sprintf("enterprises/%s/copilot/metrics/reports/user-teams-1-day", connection.Enterprise) + urlTemplate = copilotAPIPath("enterprises", connection.Enterprise, "copilot/metrics/reports/user-teams-1-day") } else if connection.Organization != "" { - urlTemplate = fmt.Sprintf("orgs/%s/copilot/metrics/reports/user-teams-1-day", connection.Organization) + urlTemplate = copilotAPIPath("orgs", connection.Organization, "copilot/metrics/reports/user-teams-1-day") } else { return nil } diff --git a/backend/plugins/github/api/scope_duplicates_api.go b/backend/plugins/github/api/scope_duplicates_api.go new file mode 100644 index 00000000000..e3cc2312b09 --- /dev/null +++ b/backend/plugins/github/api/scope_duplicates_api.go @@ -0,0 +1,205 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "net/http" + "strconv" + "strings" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +// ScopeDuplicateConnection is a connection that shares a repository scope. +type ScopeDuplicateConnection struct { + ConnectionId uint64 `json:"connectionId"` + ConnectionName string `json:"connectionName"` +} + +// ScopeDuplicateGroup is one repository that appears under multiple connections +// (diagnostics) or already exists under another connection (pre-add check). +type ScopeDuplicateGroup struct { + GithubId int `json:"githubId"` + HTMLUrl string `json:"htmlUrl"` + FullName string `json:"fullName"` + Connections []ScopeDuplicateConnection `json:"connections"` +} + +// ScopeDuplicatesOutput is the response body for GetScopeDuplicates. +type ScopeDuplicatesOutput struct { + Duplicates []ScopeDuplicateGroup `json:"duplicates"` +} + +// scopeDuplicateRow is one joined row from the scoped SQL query. +type scopeDuplicateRow struct { + GithubId int `gorm:"column:github_id"` + HTMLUrl string `gorm:"column:html_url"` + FullName string `gorm:"column:full_name"` + ConnectionId uint64 `gorm:"column:connection_id"` + ConnectionName string `gorm:"column:connection_name"` +} + +// GetScopeDuplicates returns GitHub repositories registered under more than one +// connection, or (with connectionId + githubIds) candidates already present on +// other connections. +// @Summary Find GitHub scopes duplicated across connections +// @Description Diagnostics: groups where the same githubId appears on more than one connection. +// @Description Pre-add check: pass connectionId and githubIds to find candidates already registered elsewhere. +// @Tags plugins/github +// @Param connectionId query int false "Current connection id (pre-add check)" +// @Param githubIds query string false "Comma-separated GitHub repo ids to check (pre-add check)" +// @Success 200 {object} ScopeDuplicatesOutput +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/github/scope-duplicates [GET] +func GetScopeDuplicates(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connectionId, githubIds, err := parseScopeDuplicateQuery(input) + if err != nil { + return nil, err + } + + // Pre-add check with an empty selection: nothing to warn about. + if connectionId != nil && len(githubIds) == 0 { + return &plugin.ApiResourceOutput{ + Body: ScopeDuplicatesOutput{Duplicates: []ScopeDuplicateGroup{}}, + Status: http.StatusOK, + }, nil + } + + rows, err := queryScopeDuplicateRows(basicRes.GetDal(), connectionId, githubIds) + if err != nil { + return nil, err + } + + return &plugin.ApiResourceOutput{ + Body: ScopeDuplicatesOutput{Duplicates: groupScopeDuplicateRows(rows)}, + Status: http.StatusOK, + }, nil +} + +func parseScopeDuplicateQuery(input *plugin.ApiResourceInput) (*uint64, []int, errors.Error) { + var connectionId *uint64 + if v := input.Query.Get("connectionId"); v != "" { + id, err := strconv.ParseUint(v, 10, 64) + if err != nil { + return nil, nil, errors.BadInput.Wrap(err, "invalid connectionId") + } + connectionId = &id + } + + var githubIds []int + if v := input.Query.Get("githubIds"); v != "" { + for _, part := range strings.Split(v, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + id, err := strconv.Atoi(part) + if err != nil { + return nil, nil, errors.BadInput.Wrap(err, "invalid githubIds") + } + githubIds = append(githubIds, id) + } + } + + if len(githubIds) > 0 && connectionId == nil { + return nil, nil, errors.BadInput.New("connectionId is required when githubIds is provided") + } + + return connectionId, githubIds, nil +} + +// queryScopeDuplicateRows loads only the rows needed for the requested mode. +// Check mode: selected githubIds on any connection other than connectionId. +// Diagnostics: githubIds that already appear on more than one connection. +func queryScopeDuplicateRows(db dal.Dal, connectionId *uint64, githubIds []int) ([]scopeDuplicateRow, errors.Error) { + clauses := []dal.Clause{ + dal.Select("r.github_id, r.html_url, r.full_name, r.connection_id, c.name AS connection_name"), + dal.From("_tool_github_repos r"), + dal.Join("INNER JOIN _tool_github_connections c ON c.id = r.connection_id"), + dal.Orderby("r.github_id ASC, r.connection_id ASC"), + } + + if connectionId != nil { + clauses = append(clauses, dal.Where( + "r.github_id IN ? AND r.connection_id != ?", + githubIds, + *connectionId, + )) + } else { + clauses = append(clauses, dal.Where(`r.github_id IN ( + SELECT github_id FROM _tool_github_repos + GROUP BY github_id + HAVING COUNT(DISTINCT connection_id) > 1 + )`)) + } + + var rows []scopeDuplicateRow + if err := db.All(&rows, clauses...); err != nil { + return nil, err + } + return rows, nil +} + +// groupScopeDuplicateRows collapses already-filtered SQL rows into API groups. +func groupScopeDuplicateRows(rows []scopeDuplicateRow) []ScopeDuplicateGroup { + if len(rows) == 0 { + return []ScopeDuplicateGroup{} + } + + result := make([]ScopeDuplicateGroup, 0) + var current *ScopeDuplicateGroup + seenConns := make(map[uint64]struct{}) + + flush := func() { + if current != nil { + result = append(result, *current) + } + } + + for _, row := range rows { + if current == nil || current.GithubId != row.GithubId { + flush() + current = &ScopeDuplicateGroup{ + GithubId: row.GithubId, + HTMLUrl: row.HTMLUrl, + FullName: row.FullName, + Connections: make([]ScopeDuplicateConnection, 0, 2), + } + seenConns = make(map[uint64]struct{}) + } + if current.HTMLUrl == "" && row.HTMLUrl != "" { + current.HTMLUrl = row.HTMLUrl + } + if current.FullName == "" && row.FullName != "" { + current.FullName = row.FullName + } + if _, ok := seenConns[row.ConnectionId]; ok { + continue + } + seenConns[row.ConnectionId] = struct{}{} + current.Connections = append(current.Connections, ScopeDuplicateConnection{ + ConnectionId: row.ConnectionId, + ConnectionName: row.ConnectionName, + }) + } + flush() + return result +} diff --git a/backend/plugins/github/api/scope_duplicates_api_test.go b/backend/plugins/github/api/scope_duplicates_api_test.go new file mode 100644 index 00000000000..87cc33456e3 --- /dev/null +++ b/backend/plugins/github/api/scope_duplicates_api_test.go @@ -0,0 +1,114 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "net/url" + "testing" + + "github.com/apache/incubator-devlake/core/plugin" + "github.com/stretchr/testify/assert" +) + +func TestGroupScopeDuplicateRows_Empty(t *testing.T) { + assert.Empty(t, groupScopeDuplicateRows(nil)) + assert.Empty(t, groupScopeDuplicateRows([]scopeDuplicateRow{})) +} + +func TestGroupScopeDuplicateRows_GroupsConnections(t *testing.T) { + rows := []scopeDuplicateRow{ + {GithubId: 100, HTMLUrl: "https://github.com/o/a", FullName: "o/a", ConnectionId: 1, ConnectionName: "GitHub Production"}, + {GithubId: 100, HTMLUrl: "https://github.com/o/a", FullName: "o/a", ConnectionId: 2, ConnectionName: "GitHub Staging"}, + {GithubId: 200, HTMLUrl: "https://github.com/o/b", FullName: "o/b", ConnectionId: 3, ConnectionName: "Other"}, + } + + got := groupScopeDuplicateRows(rows) + assert.Equal(t, []ScopeDuplicateGroup{ + { + GithubId: 100, + HTMLUrl: "https://github.com/o/a", + FullName: "o/a", + Connections: []ScopeDuplicateConnection{ + {ConnectionId: 1, ConnectionName: "GitHub Production"}, + {ConnectionId: 2, ConnectionName: "GitHub Staging"}, + }, + }, + { + GithubId: 200, + HTMLUrl: "https://github.com/o/b", + FullName: "o/b", + Connections: []ScopeDuplicateConnection{ + {ConnectionId: 3, ConnectionName: "Other"}, + }, + }, + }, got) +} + +func TestGroupScopeDuplicateRows_DedupesSameConnection(t *testing.T) { + rows := []scopeDuplicateRow{ + {GithubId: 100, FullName: "o/a", ConnectionId: 1, ConnectionName: "Prod"}, + {GithubId: 100, FullName: "o/a", ConnectionId: 1, ConnectionName: "Prod"}, + } + + got := groupScopeDuplicateRows(rows) + assert.Len(t, got, 1) + assert.Equal(t, []ScopeDuplicateConnection{ + {ConnectionId: 1, ConnectionName: "Prod"}, + }, got[0].Connections) +} + +func TestGroupScopeDuplicateRows_FillsMissingLabels(t *testing.T) { + rows := []scopeDuplicateRow{ + {GithubId: 100, ConnectionId: 1, ConnectionName: "A"}, + {GithubId: 100, HTMLUrl: "https://github.com/o/a", FullName: "o/a", ConnectionId: 2, ConnectionName: "B"}, + } + + got := groupScopeDuplicateRows(rows) + assert.Len(t, got, 1) + assert.Equal(t, "https://github.com/o/a", got[0].HTMLUrl) + assert.Equal(t, "o/a", got[0].FullName) +} + +func TestParseScopeDuplicateQuery(t *testing.T) { + input := &plugin.ApiResourceInput{Query: url.Values{}} + connId, ids, err := parseScopeDuplicateQuery(input) + assert.Nil(t, err) + assert.Nil(t, connId) + assert.Empty(t, ids) + + input = &plugin.ApiResourceInput{Query: url.Values{ + "connectionId": []string{"3"}, + "githubIds": []string{"10, 20,30"}, + }} + connId, ids, err = parseScopeDuplicateQuery(input) + assert.Nil(t, err) + assert.Equal(t, uint64(3), *connId) + assert.Equal(t, []int{10, 20, 30}, ids) + + input = &plugin.ApiResourceInput{Query: url.Values{ + "githubIds": []string{"10"}, + }} + _, _, err = parseScopeDuplicateQuery(input) + assert.Error(t, err) + + input = &plugin.ApiResourceInput{Query: url.Values{ + "connectionId": []string{"abc"}, + }} + _, _, err = parseScopeDuplicateQuery(input) + assert.Error(t, err) +} diff --git a/backend/plugins/github/impl/impl.go b/backend/plugins/github/impl/impl.go index e09779a3dcd..ada24da0da8 100644 --- a/backend/plugins/github/impl/impl.go +++ b/backend/plugins/github/impl/impl.go @@ -225,6 +225,9 @@ func (p Github) ApiResources() map[string]map[string]plugin.ApiResourceHandler { "scope-config/:scopeConfigId/projects": { "GET": api.GetProjectsByScopeConfig, }, + "scope-duplicates": { + "GET": api.GetScopeDuplicates, + }, } } diff --git a/backend/plugins/gitlab/tasks/shared.go b/backend/plugins/gitlab/tasks/shared.go index 6621ba3f2fd..eac88b66f67 100644 --- a/backend/plugins/gitlab/tasks/shared.go +++ b/backend/plugins/gitlab/tasks/shared.go @@ -195,7 +195,14 @@ func GetMergeRequestsIterator(taskCtx plugin.SubTaskContext, apiCollector *api.S } if apiCollector != nil { if apiCollector.GetSince() != nil { - clauses = append(clauses, dal.Where("gitlab_updated_at > ?", *apiCollector.GetSince())) + // Filter by the LATER of gitlab_updated_at or commit_updated_at. + // Using only gitlab_updated_at misses MRs where new commits were pushed + // without the MR itself being updated (e.g. force-pushed commits). + // COALESCE handles MRs with no recorded commit_updated_at. + clauses = append(clauses, dal.Where( + `GREATEST(gmr.gitlab_updated_at, COALESCE(gmr.commit_updated_at, gmr.gitlab_updated_at)) > ?`, + *apiCollector.GetSince(), + )) } } // construct the input iterator diff --git a/backend/plugins/incidentio/api/blueprint_v200.go b/backend/plugins/incidentio/api/blueprint_v200.go new file mode 100644 index 00000000000..81b3bee0c75 --- /dev/null +++ b/backend/plugins/incidentio/api/blueprint_v200.go @@ -0,0 +1,103 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/utils" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/incidentio/models" + "github.com/apache/incubator-devlake/plugins/incidentio/tasks" +) + +func MakeDataSourcePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + connectionId uint64, + bpScopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + connection, err := dsHelper.ConnSrv.FindByPk(connectionId) + if err != nil { + return nil, nil, err + } + scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes) + if err != nil { + return nil, nil, err + } + plan, err := makePipelinePlanV200(subtaskMetas, scopeDetails, connection) + if err != nil { + return nil, nil, err + } + scopes, err := makeScopesV200(scopeDetails, connection) + return plan, scopes, err +} + +func makePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + scopeDetails []*srvhelper.ScopeDetail[models.IncidentType, models.IncidentioScopeConfig], + connection *models.IncidentioConnection, +) (coreModels.PipelinePlan, errors.Error) { + plan := make(coreModels.PipelinePlan, len(scopeDetails)) + for i, scopeDetail := range scopeDetails { + stage := plan[i] + if stage == nil { + stage = coreModels.PipelineStage{} + } + + scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig + task, err := api.MakePipelinePlanTask( + "incidentio", + subtaskMetas, + scopeConfig.Entities, + tasks.IncidentioOptions{ + ConnectionId: connection.ID, + IncidentTypeId: scope.Id, + }, + ) + if err != nil { + return nil, err + } + stage = append(stage, task) + plan[i] = stage + } + + return plan, nil +} + +func makeScopesV200( + scopeDetails []*srvhelper.ScopeDetail[models.IncidentType, models.IncidentioScopeConfig], + connection *models.IncidentioConnection, +) ([]plugin.Scope, errors.Error) { + scopes := make([]plugin.Scope, 0, len(scopeDetails)) + + idgen := didgen.NewDomainIdGenerator(&models.IncidentType{}) + for _, scopeDetail := range scopeDetails { + scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig + id := idgen.Generate(connection.ID, scope.Id) + + if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_TICKET) { + scopes = append(scopes, ticket.NewBoard(id, scope.Name)) + } + } + + return scopes, nil +} diff --git a/backend/plugins/incidentio/api/connection_api.go b/backend/plugins/incidentio/api/connection_api.go new file mode 100644 index 00000000000..c7c77bad0d5 --- /dev/null +++ b/backend/plugins/incidentio/api/connection_api.go @@ -0,0 +1,155 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/incidentio/models" +) + +func testConnection(ctx context.Context, connection models.IncidentioConn) (*plugin.ApiResourceOutput, errors.Error) { + if vld != nil { + if err := vld.Struct(connection); err != nil { + return nil, errors.Default.Wrap(err, "error validating target") + } + } + apiClient, err := api.NewApiClientFromConnection(ctx, basicRes, &connection) + if err != nil { + return nil, err + } + response, err := apiClient.Get("v1/incident_types", nil, nil) + if err != nil { + return nil, err + } + if response.StatusCode == http.StatusUnauthorized { + return nil, errors.HttpStatus(http.StatusBadRequest).New("StatusUnauthorized error while testing connection") + } + if response.StatusCode == http.StatusOK { + return &plugin.ApiResourceOutput{Body: nil, Status: http.StatusOK}, nil + } + return &plugin.ApiResourceOutput{Body: nil, Status: response.StatusCode}, errors.HttpStatus(response.StatusCode).Wrap(err, "could not validate connection") +} + +// TestConnection test incidentio connection +// @Summary test incidentio connection +// @Description Test incident.io Connection +// @Tags plugins/incidentio +// @Param body body models.IncidentioConn true "json body" +// @Success 200 {object} shared.ApiBody "Success" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/incidentio/test [POST] +func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + var connection models.IncidentioConn + err := api.Decode(input.Body, &connection, vld) + if err != nil { + return nil, err + } + testConnectionResult, testConnectionErr := testConnection(context.TODO(), connection) + if testConnectionErr != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, testConnectionErr) + } + return testConnectionResult, nil +} + +// TestExistingConnection test incidentio connection +// @Summary test incidentio connection +// @Description Test incident.io Connection +// @Tags plugins/incidentio +// @Param connectionId path int true "connection ID" +// @Success 200 {object} shared.ApiBody "Success" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/incidentio/connections/{connectionId}/test [POST] +func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection, err := dsHelper.ConnApi.GetMergedConnection(input) + if err != nil { + return nil, errors.BadInput.Wrap(err, "find connection from db") + } + if err := api.DecodeMapStruct(input.Body, connection, false); err != nil { + return nil, err + } + testConnectionResult, testConnectionErr := testConnection(context.TODO(), connection.IncidentioConn) + if testConnectionErr != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, testConnectionErr) + } + return testConnectionResult, nil +} + +// @Summary create incidentio connection +// @Description Create incident.io connection +// @Tags plugins/incidentio +// @Param body body models.IncidentioConnection true "json body" +// @Success 200 {object} models.IncidentioConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/incidentio/connections [POST] +func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Post(input) +} + +// @Summary patch incidentio connection +// @Description Patch incident.io connection +// @Tags plugins/incidentio +// @Param body body models.IncidentioConnection true "json body" +// @Success 200 {object} models.IncidentioConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/incidentio/connections/{connectionId} [PATCH] +func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Patch(input) +} + +// @Summary delete incidentio connection +// @Description Delete incident.io connection +// @Tags plugins/incidentio +// @Success 200 {object} models.IncidentioConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 409 {object} services.BlueprintProjectPairs "References exist to this connection" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/incidentio/connections/{connectionId} [DELETE] +func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Delete(input) +} + +// @Summary list incidentio connections +// @Description List incident.io connections +// @Tags plugins/incidentio +// @Success 200 {object} models.IncidentioConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/incidentio/connections [GET] +func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetAll(input) +} + +// @Summary get incidentio connection +// @Description Get incident.io connection +// @Tags plugins/incidentio +// @Success 200 {object} models.IncidentioConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/incidentio/connections/{connectionId} [GET] +func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetDetail(input) +} diff --git a/backend/plugins/incidentio/api/init.go b/backend/plugins/incidentio/api/init.go new file mode 100644 index 00000000000..36a9345ed70 --- /dev/null +++ b/backend/plugins/incidentio/api/init.go @@ -0,0 +1,55 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/incidentio/models" + "github.com/go-playground/validator/v10" +) + +var vld *validator.Validate +var basicRes context.BasicRes + +var dsHelper *api.DsHelper[models.IncidentioConnection, models.IncidentType, models.IncidentioScopeConfig] +var raProxy *api.DsRemoteApiProxyHelper[models.IncidentioConnection] +var raScopeList *api.DsRemoteApiScopeListHelper[models.IncidentioConnection, models.IncidentType, IncidentioRemotePagination] + +var raScopeSearch *api.DsRemoteApiScopeSearchHelper[models.IncidentioConnection, models.IncidentType] + +func Init(br context.BasicRes, p plugin.PluginMeta) { + vld = validator.New() + basicRes = br + dsHelper = api.NewDataSourceHelper[ + models.IncidentioConnection, models.IncidentType, models.IncidentioScopeConfig, + ]( + br, + p.Name(), + []string{"name"}, + func(c models.IncidentioConnection) models.IncidentioConnection { + return c.Sanitize() + }, + nil, + nil, + ) + raProxy = api.NewDsRemoteApiProxyHelper[models.IncidentioConnection](dsHelper.ConnApi.ModelApiHelper) + raScopeList = api.NewDsRemoteApiScopeListHelper[models.IncidentioConnection, models.IncidentType, IncidentioRemotePagination](raProxy, listIncidentioRemoteScopes) + raScopeSearch = api.NewDsRemoteApiScopeSearchHelper[models.IncidentioConnection, models.IncidentType](raProxy, searchIncidentioRemoteScopes) +} diff --git a/backend/plugins/incidentio/api/remote_api.go b/backend/plugins/incidentio/api/remote_api.go new file mode 100644 index 00000000000..6c8ca5318b0 --- /dev/null +++ b/backend/plugins/incidentio/api/remote_api.go @@ -0,0 +1,155 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "net/http" + "strings" + "time" + + "github.com/apache/incubator-devlake/core/models/common" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + dsmodels "github.com/apache/incubator-devlake/helpers/pluginhelper/api/models" + "github.com/apache/incubator-devlake/plugins/incidentio/models" +) + +type IncidentioRemotePagination struct { + Page int `json:"page"` + PerPage int `json:"per_page"` +} + +type IncidentTypesResponse struct { + IncidentTypes []struct { + Id string `json:"id"` + Name string `json:"name"` + Description *string `json:"description"` + CreatedAt *time.Time `json:"created_at"` + } `json:"incident_types"` +} + +// queryIncidentioRemoteScopes lists incident types as scopes. The +// endpoint is not paginated and returns the full list, so search is +// applied client-side and there is never a next page. +func queryIncidentioRemoteScopes( + apiClient plugin.ApiClient, + _ string, + page IncidentioRemotePagination, + search string, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.IncidentType], + nextPage *IncidentioRemotePagination, + err errors.Error, +) { + var res *http.Response + res, err = apiClient.Get("v1/incident_types", nil, nil) + if err != nil { + return + } + response := &IncidentTypesResponse{} + err = api.UnmarshalResponse(res, response) + if err != nil { + return + } + for _, item := range response.IncidentTypes { + if search != "" && !strings.Contains(strings.ToLower(item.Name), strings.ToLower(search)) { + continue + } + entry := dsmodels.DsRemoteApiScopeListEntry[models.IncidentType]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + Id: item.Id, + Name: item.Name, + FullName: item.Name, + Data: &models.IncidentType{ + Id: item.Id, + Name: item.Name, + Scope: common.Scope{ + NoPKModel: common.NoPKModel{}, + }, + }, + } + if item.CreatedAt != nil { + entry.Data.Scope.NoPKModel.CreatedAt = *item.CreatedAt + } + children = append(children, entry) + } + + return +} + +func listIncidentioRemoteScopes( + connection *models.IncidentioConnection, + apiClient plugin.ApiClient, + groupId string, + page IncidentioRemotePagination, +) ( + []dsmodels.DsRemoteApiScopeListEntry[models.IncidentType], + *IncidentioRemotePagination, + errors.Error, +) { + return queryIncidentioRemoteScopes(apiClient, groupId, page, "") +} + +func searchIncidentioRemoteScopes( + apiClient plugin.ApiClient, + params *dsmodels.DsRemoteApiScopeSearchParams, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.IncidentType], + err errors.Error, +) { + children, _, err = queryIncidentioRemoteScopes(apiClient, "", IncidentioRemotePagination{ + Page: params.Page, + PerPage: params.PageSize, + }, params.Search) + return +} + +// RemoteScopes list all available scopes (incident types) for this connection +// @Summary list all available scopes (incident types) for this connection +// @Description list all available scopes (incident types) for this connection +// @Tags plugins/incidentio +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param groupId query string false "group ID" +// @Param pageToken query string false "page Token" +// @Success 200 {object} RemoteScopesOutput +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/incidentio/connections/{connectionId}/remote-scopes [GET] +func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raScopeList.Get(input) +} + +// SearchRemoteScopes use the Search API and only return project +// @Summary use the Search API and only return project +// @Description use the Search API and only return project +// @Tags plugins/incidentio +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param search query string false "search" +// @Param page query int false "page number" +// @Param pageSize query int false "page size per page" +// @Success 200 {object} SearchRemoteScopesOutput +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/incidentio/connections/{connectionId}/search-remote-scopes [GET] +func SearchRemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raScopeSearch.Get(input) +} diff --git a/backend/plugins/incidentio/api/scope_api.go b/backend/plugins/incidentio/api/scope_api.go new file mode 100644 index 00000000000..1d237812930 --- /dev/null +++ b/backend/plugins/incidentio/api/scope_api.go @@ -0,0 +1,107 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/incidentio/models" +) + +type PutScopesReqBody api.PutScopesReqBody[models.IncidentType] +type ScopeDetail api.ScopeDetail[models.IncidentType, models.IncidentioScopeConfig] + +// PutScopes create or update incidentio incident type +// @Summary create or update incidentio incident type +// @Description Create or update incidentio incident type +// @Tags plugins/incidentio +// @Accept application/json +// @Param connectionId path int true "connection ID" +// @Param scope body ScopeReq true "json" +// @Success 200 {object} []ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/incidentio/connections/{connectionId}/scopes [PUT] +func PutScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.PutMultiple(input) +} + +// PatchScope patch to incidentio incident type +// @Summary patch to incidentio incident type +// @Description patch to incidentio incident type +// @Tags plugins/incidentio +// @Accept application/json +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Param scope body models.IncidentType true "json" +// @Success 200 {object} models.IncidentType +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/incidentio/connections/{connectionId}/scopes/{scopeId} [PATCH] +func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Patch(input) +} + +// GetScopeList get incidentio incident types +// @Summary get incidentio incident types +// @Description get incidentio incident types +// @Tags plugins/incidentio +// @Param connectionId path int true "connection ID" +// @Param searchTerm query string false "search term for scope name" +// @Param pageSize query int false "page size, default 50" +// @Param page query int false "page size, default 1" +// @Param blueprints query bool false "also return blueprints using these scopes as part of the payload" +// @Success 200 {object} []ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/incidentio/connections/{connectionId}/scopes/ [GET] +func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetPage(input) +} + +// GetScope get one incidentio incident type +// @Summary get one incidentio incident type +// @Description get one incidentio incident type +// @Tags plugins/incidentio +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Param blueprints query bool false "also return blueprints using this scope as part of the payload" +// @Success 200 {object} ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/incidentio/connections/{connectionId}/scopes/{scopeId} [GET] +func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeDetail(input) +} + +// DeleteScope delete plugin data associated with the scope and optionally the scope itself +// @Summary delete plugin data associated with the scope and optionally the scope itself +// @Description delete data associated with plugin scope +// @Tags plugins/incidentio +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Param delete_data_only query bool false "Only delete the scope data, not the scope itself" +// @Success 200 +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 409 {object} api.ScopeRefDoc "References exist to this scope" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/incidentio/connections/{connectionId}/scopes/{scopeId} [DELETE] +func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Delete(input) +} diff --git a/backend/plugins/incidentio/api/scope_state_api.go b/backend/plugins/incidentio/api/scope_state_api.go new file mode 100644 index 00000000000..b8c6cb6c509 --- /dev/null +++ b/backend/plugins/incidentio/api/scope_state_api.go @@ -0,0 +1,37 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +// GetScopeLatestSyncState get one incidentio incident type's latest sync state +// @Summary get one incidentio incident type's latest sync state +// @Description get one incidentio incident type's latest sync state +// @Tags plugins/incidentio +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Success 200 {object} []models.LatestSyncState +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/incidentio/connections/{connectionId}/scopes/{scopeId}/latest-sync-state [GET] +func GetScopeLatestSyncState(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeLatestSyncState(input) +} diff --git a/backend/plugins/incidentio/api/swagger.go b/backend/plugins/incidentio/api/swagger.go new file mode 100644 index 00000000000..91691513c69 --- /dev/null +++ b/backend/plugins/incidentio/api/swagger.go @@ -0,0 +1,32 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/plugins/incidentio/tasks" +) + +type IncidentioTaskOptions tasks.IncidentioOptions + +// @Summary incidentio task options for pipelines +// @Description This is a dummy API to demonstrate the available task options for incidentio pipelines +// @Tags plugins/incidentio +// @Accept application/json +// @Param pipeline body IncidentioTaskOptions true "json" +// @Router /pipelines/incidentio/pipeline-task [post] +func _() {} diff --git a/backend/plugins/incidentio/e2e/incident_test.go b/backend/plugins/incidentio/e2e/incident_test.go new file mode 100644 index 00000000000..7f19a34aab9 --- /dev/null +++ b/backend/plugins/incidentio/e2e/incident_test.go @@ -0,0 +1,108 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/incidentio/impl" + "github.com/apache/incubator-devlake/plugins/incidentio/models" + "github.com/apache/incubator-devlake/plugins/incidentio/tasks" + "github.com/stretchr/testify/require" +) + +func TestIncidentDataFlow(t *testing.T) { + var plugin impl.Incidentio + dataflowTester := e2ehelper.NewDataFlowTester(t, "incidentio", plugin) + options := tasks.IncidentioOptions{ + ConnectionId: 1, + IncidentTypeId: "type_01", + IncidentTypeName: "Default", + } + taskData := &tasks.IncidentioTaskData{ + Options: &options, + } + + // scope + dataflowTester.FlushTabler(&models.IncidentType{}) + incidentType := models.IncidentType{ + Scope: common.Scope{ + ConnectionId: options.ConnectionId, + }, + Id: options.IncidentTypeId, + Name: options.IncidentTypeName, + } + require.NoError(t, dataflowTester.Dal.CreateOrUpdate(&incidentType)) + + // import raw data table + dataflowTester.ImportCsvIntoRawTable( + "./raw_tables/_raw_incidentio_incidents.csv", + "_raw_incidentio_incidents", + ) + + // verify extraction + dataflowTester.FlushTabler(&models.Incident{}) + dataflowTester.Subtask(tasks.ExtractIncidentsMeta, taskData) + dataflowTester.VerifyTableWithOptions( + models.IncidentType{}, + e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_incidentio_incident_types.csv", + IgnoreTypes: []any{common.Scope{}}, + }, + ) + dataflowTester.VerifyTableWithOptions( + models.Incident{}, + e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_incidentio_incidents.csv", + IgnoreTypes: []any{common.NoPKModel{}}, + }, + ) + + // verify conversion + dataflowTester.FlushTabler(&ticket.Board{}) + dataflowTester.Subtask(tasks.ConvertIncidentTypesMeta, taskData) + dataflowTester.VerifyTableWithOptions( + ticket.Board{}, + e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/boards.csv", + IgnoreTypes: []any{common.NoPKModel{}}, + }, + ) + + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.Subtask(tasks.ConvertIncidentsMeta, taskData) + dataflowTester.VerifyTableWithOptions( + ticket.Issue{}, + e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issues.csv", + IgnoreTypes: []any{common.NoPKModel{}}, + IgnoreFields: []string{"original_project"}, + }, + ) + dataflowTester.VerifyTableWithOptions( + ticket.BoardIssue{}, + e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/board_issues.csv", + IgnoreTypes: []any{common.NoPKModel{}}, + }, + ) +} diff --git a/backend/plugins/incidentio/e2e/raw_tables/_raw_incidentio_incidents.csv b/backend/plugins/incidentio/e2e/raw_tables/_raw_incidentio_incidents.csv new file mode 100644 index 00000000000..79882c03aa5 --- /dev/null +++ b/backend/plugins/incidentio/e2e/raw_tables/_raw_incidentio_incidents.csv @@ -0,0 +1,6 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""ScopeId"":""type_01""}","{""id"":""inc_01"",""reference"":""INC-101"",""name"":""Payment processor outage"",""summary"":""Payments are timing out"",""permalink"":""https://app.incident.io/example/incidents/101"",""mode"":""standard"",""created_at"":""2026-05-01T09:55:00Z"",""updated_at"":""2026-05-01T11:31:00Z"",""incident_status"":{""name"":""Closed"",""category"":""closed""},""severity"":{""id"":""sev_3"",""name"":""Critical"",""rank"":3},""incident_type"":{""id"":""type_01"",""name"":""Default""},""creator"":{""user"":{""id"":""u1"",""name"":""Alice"",""email"":""alice@example.com""}},""incident_timestamp_values"":[{""incident_timestamp"":{""id"":""ts_1"",""name"":""Reported at""},""value"":{""value"":""2026-05-01T09:55:00Z""}},{""incident_timestamp"":{""id"":""ts_2"",""name"":""Declared at""},""value"":{""value"":""2026-05-01T10:00:00Z""}},{""incident_timestamp"":{""id"":""ts_3"",""name"":""Resolved at""},""value"":{""value"":""2026-05-01T11:30:00Z""}},{""incident_timestamp"":{""id"":""ts_4"",""name"":""Closed at""},""value"":{""value"":""2026-05-01T11:31:00Z""}}]}",,null,2026-05-01T11:31:00.000+00:00 +2,"{""ConnectionId"":1,""ScopeId"":""type_01""}","{""id"":""inc_02"",""reference"":""INC-102"",""name"":""Latency spike"",""summary"":""p99 latency above SLO"",""permalink"":""https://app.incident.io/example/incidents/102"",""mode"":""retrospective"",""created_at"":""2026-05-02T09:00:00Z"",""updated_at"":""2026-05-02T09:10:00Z"",""incident_status"":{""name"":""Closed"",""category"":""closed""},""severity"":{""id"":""sev_2"",""name"":""Major"",""rank"":2},""incident_type"":{""id"":""type_01"",""name"":""Default""},""creator"":{""user"":{""id"":""u2"",""name"":""Bob"",""email"":""bob@example.com""}},""incident_timestamp_values"":[{""incident_timestamp"":{""id"":""ts_2"",""name"":""Declared at""},""value"":{""value"":""2026-05-02T09:05:00Z""}},{""incident_timestamp"":{""id"":""ts_3"",""name"":""Resolved at""},""value"":{""value"":""2026-05-01T18:00:00Z""}},{""incident_timestamp"":{""id"":""ts_4"",""name"":""Closed at""},""value"":{""value"":""2026-05-02T09:10:00Z""}}]}",,null,2026-05-02T09:10:00.000+00:00 +3,"{""ConnectionId"":1,""ScopeId"":""type_01""}","{""id"":""inc_03"",""reference"":""INC-103"",""name"":""Practice run"",""summary"":""Game day exercise"",""permalink"":""https://app.incident.io/example/incidents/103"",""mode"":""test"",""created_at"":""2026-05-03T12:00:00Z"",""updated_at"":""2026-05-03T12:30:00Z"",""incident_status"":{""name"":""Closed"",""category"":""closed""},""severity"":{""id"":""sev_1"",""name"":""Minor"",""rank"":1},""incident_type"":{""id"":""type_01"",""name"":""Default""},""incident_timestamp_values"":[{""incident_timestamp"":{""id"":""ts_2"",""name"":""Declared at""},""value"":{""value"":""2026-05-03T12:00:00Z""}}]}",,null,2026-05-03T12:30:00.000+00:00 +4,"{""ConnectionId"":1,""ScopeId"":""type_01""}","{""id"":""inc_04"",""reference"":""INC-104"",""name"":""Queue backlog"",""summary"":""Worker queue backed up"",""permalink"":""https://app.incident.io/example/incidents/104"",""mode"":""standard"",""created_at"":""2026-05-04T08:00:00Z"",""updated_at"":""2026-05-04T08:31:00Z"",""incident_status"":{""name"":""Closed"",""category"":""closed""},""severity"":{""id"":""sev_1"",""name"":""Minor"",""rank"":1},""incident_type"":{""id"":""type_01"",""name"":""Default""},""creator"":{""user"":{""id"":""u1"",""name"":""Alice"",""email"":""alice@example.com""}},""incident_timestamp_values"":[{""incident_timestamp"":{""id"":""ts_2"",""name"":""Declared at""},""value"":{""value"":""2026-05-04T08:00:00Z""}},{""incident_timestamp"":{""id"":""ts_3"",""name"":""Resolved at""},""value"":null},{""incident_timestamp"":{""id"":""ts_4"",""name"":""Closed at""},""value"":{""value"":""2026-05-04T08:30:00Z""}}]}",,null,2026-05-04T08:31:00.000+00:00 +5,"{""ConnectionId"":1,""ScopeId"":""type_01""}","{""id"":""inc_05"",""reference"":""INC-105"",""name"":""Security review"",""summary"":""Belongs to the Security incident type"",""permalink"":""https://app.incident.io/example/incidents/105"",""mode"":""standard"",""created_at"":""2026-05-05T14:00:00Z"",""updated_at"":""2026-05-05T14:05:00Z"",""incident_status"":{""name"":""Investigating"",""category"":""active""},""severity"":{""id"":""sev_2"",""name"":""Major"",""rank"":2},""incident_type"":{""id"":""type_99"",""name"":""Security""},""incident_timestamp_values"":[{""incident_timestamp"":{""id"":""ts_2"",""name"":""Declared at""},""value"":{""value"":""2026-05-05T14:00:00Z""}}]}",,null,2026-05-05T14:05:00.000+00:00 diff --git a/backend/plugins/incidentio/e2e/snapshot_tables/_tool_incidentio_incident_types.csv b/backend/plugins/incidentio/e2e/snapshot_tables/_tool_incidentio_incident_types.csv new file mode 100644 index 00000000000..513ef7885c9 --- /dev/null +++ b/backend/plugins/incidentio/e2e/snapshot_tables/_tool_incidentio_incident_types.csv @@ -0,0 +1,2 @@ +connection_id,id,name +1,type_01,Default diff --git a/backend/plugins/incidentio/e2e/snapshot_tables/_tool_incidentio_incidents.csv b/backend/plugins/incidentio/e2e/snapshot_tables/_tool_incidentio_incidents.csv new file mode 100644 index 00000000000..74185e7851d --- /dev/null +++ b/backend/plugins/incidentio/e2e/snapshot_tables/_tool_incidentio_incidents.csv @@ -0,0 +1,4 @@ +connection_id,id,reference,name,summary,url,mode,status_name,status_category,severity_name,severity_rank,incident_type_id,created_date,updated_date,declared_date,resolved_date +1,inc_01,INC-101,Payment processor outage,Payments are timing out,https://app.incident.io/example/incidents/101,standard,Closed,closed,Critical,3,type_01,2026-05-01T09:55:00.000+00:00,2026-05-01T11:31:00.000+00:00,2026-05-01T10:00:00.000+00:00,2026-05-01T11:30:00.000+00:00 +1,inc_02,INC-102,Latency spike,p99 latency above SLO,https://app.incident.io/example/incidents/102,retrospective,Closed,closed,Major,2,type_01,2026-05-02T09:00:00.000+00:00,2026-05-02T09:10:00.000+00:00,2026-05-02T09:05:00.000+00:00,2026-05-01T18:00:00.000+00:00 +1,inc_04,INC-104,Queue backlog,Worker queue backed up,https://app.incident.io/example/incidents/104,standard,Closed,closed,Minor,1,type_01,2026-05-04T08:00:00.000+00:00,2026-05-04T08:31:00.000+00:00,2026-05-04T08:00:00.000+00:00,2026-05-04T08:30:00.000+00:00 diff --git a/backend/plugins/incidentio/e2e/snapshot_tables/board_issues.csv b/backend/plugins/incidentio/e2e/snapshot_tables/board_issues.csv new file mode 100644 index 00000000000..9d6a8d037bf --- /dev/null +++ b/backend/plugins/incidentio/e2e/snapshot_tables/board_issues.csv @@ -0,0 +1,4 @@ +board_id,issue_id +incidentio:IncidentType:1:type_01,incidentio:Incident:1:inc_01 +incidentio:IncidentType:1:type_01,incidentio:Incident:1:inc_02 +incidentio:IncidentType:1:type_01,incidentio:Incident:1:inc_04 diff --git a/backend/plugins/incidentio/e2e/snapshot_tables/boards.csv b/backend/plugins/incidentio/e2e/snapshot_tables/boards.csv new file mode 100644 index 00000000000..4026a61c54d --- /dev/null +++ b/backend/plugins/incidentio/e2e/snapshot_tables/boards.csv @@ -0,0 +1,2 @@ +id,name,description,url,created_date,type +incidentio:IncidentType:1:type_01,Default,,,, diff --git a/backend/plugins/incidentio/e2e/snapshot_tables/issues.csv b/backend/plugins/incidentio/e2e/snapshot_tables/issues.csv new file mode 100644 index 00000000000..7833d9cc63f --- /dev/null +++ b/backend/plugins/incidentio/e2e/snapshot_tables/issues.csv @@ -0,0 +1,4 @@ +id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,parent_issue_id,priority,severity,urgency,component,is_subtask,due_date,fix_versions +incidentio:Incident:1:inc_01,https://app.incident.io/example/incidents/101,,INC-101,Payment processor outage,Payments are timing out,,INCIDENT,,DONE,Closed,,2026-05-01T11:30:00.000+00:00,2026-05-01T10:00:00.000+00:00,2026-05-01T11:31:00.000+00:00,90,,,,,,,,,,Critical,,,0,, +incidentio:Incident:1:inc_02,https://app.incident.io/example/incidents/102,,INC-102,Latency spike,p99 latency above SLO,,INCIDENT,,DONE,Closed,,2026-05-01T18:00:00.000+00:00,2026-05-02T09:05:00.000+00:00,2026-05-02T09:10:00.000+00:00,,,,,,,,,,,Major,,,0,, +incidentio:Incident:1:inc_04,https://app.incident.io/example/incidents/104,,INC-104,Queue backlog,Worker queue backed up,,INCIDENT,,DONE,Closed,,2026-05-04T08:30:00.000+00:00,2026-05-04T08:00:00.000+00:00,2026-05-04T08:31:00.000+00:00,30,,,,,,,,,,Minor,,,0,, diff --git a/backend/plugins/incidentio/impl/impl.go b/backend/plugins/incidentio/impl/impl.go new file mode 100644 index 00000000000..3b36e6b3f6e --- /dev/null +++ b/backend/plugins/incidentio/impl/impl.go @@ -0,0 +1,187 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "fmt" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/incidentio/api" + "github.com/apache/incubator-devlake/plugins/incidentio/models" + "github.com/apache/incubator-devlake/plugins/incidentio/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/incidentio/tasks" +) + +// make sure interface is implemented + +var _ interface { + plugin.PluginMeta + plugin.PluginInit + plugin.PluginTask + plugin.PluginApi + plugin.PluginModel + plugin.DataSourcePluginBlueprintV200 + plugin.CloseablePluginTask + plugin.PluginSource +} = (*Incidentio)(nil) + +type Incidentio struct{} + +func (p Incidentio) Description() string { + return "collect incident.io incident data" +} + +func (p Incidentio) Name() string { + return "incidentio" +} + +func (p Incidentio) Init(basicRes context.BasicRes) errors.Error { + api.Init(basicRes, p) + return nil +} + +func (p Incidentio) Connection() dal.Tabler { + return &models.IncidentioConnection{} +} + +func (p Incidentio) Scope() plugin.ToolLayerScope { + return &models.IncidentType{} +} + +func (p Incidentio) ScopeConfig() dal.Tabler { + return &models.IncidentioScopeConfig{} +} + +func (p Incidentio) SubTaskMetas() []plugin.SubTaskMeta { + // Convert incident types before incidents so the domain Board row + // exists before the BoardIssue rows that reference it. + return []plugin.SubTaskMeta{ + tasks.CollectIncidentTypesMeta, + tasks.ExtractIncidentTypesMeta, + tasks.CollectIncidentsMeta, + tasks.ExtractIncidentsMeta, + tasks.ConvertIncidentTypesMeta, + tasks.ConvertIncidentsMeta, + } +} + +func (p Incidentio) GetTablesInfo() []dal.Tabler { + return []dal.Tabler{ + &models.IncidentType{}, + &models.Incident{}, + &models.IncidentioConnection{}, + &models.IncidentioScopeConfig{}, + } +} + +func (p Incidentio) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + op, err := tasks.DecodeAndValidateTaskOptions(options) + if err != nil { + return nil, err + } + connectionHelper := helper.NewConnectionHelper( + taskCtx, + nil, + p.Name(), + ) + connection := &models.IncidentioConnection{} + err = connectionHelper.FirstById(connection, op.ConnectionId) + if err != nil { + return nil, errors.Default.Wrap(err, "unable to get incident.io connection by the given connection ID") + } + + client, err := helper.NewApiClientFromConnection(taskCtx.GetContext(), taskCtx, connection) + if err != nil { + return nil, err + } + asyncClient, err := helper.CreateAsyncApiClient(taskCtx, client, nil) + if err != nil { + return nil, err + } + return &tasks.IncidentioTaskData{ + Options: op, + Client: asyncClient, + }, nil +} + +// RootPkgPath information lost when compiled as plugin(.so) +func (p Incidentio) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/incidentio" +} + +func (p Incidentio) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.All() +} + +func (p Incidentio) ApiResources() map[string]map[string]plugin.ApiResourceHandler { + return map[string]map[string]plugin.ApiResourceHandler{ + "test": { + "POST": api.TestConnection, + }, + "connections": { + "POST": api.PostConnections, + "GET": api.ListConnections, + }, + "connections/:connectionId": { + "GET": api.GetConnection, + "PATCH": api.PatchConnection, + "DELETE": api.DeleteConnection, + }, + "connections/:connectionId/test": { + "POST": api.TestExistingConnection, + }, + "connections/:connectionId/remote-scopes": { + "GET": api.RemoteScopes, + }, + "connections/:connectionId/search-remote-scopes": { + "GET": api.SearchRemoteScopes, + }, + "connections/:connectionId/scopes": { + "GET": api.GetScopeList, + "PUT": api.PutScopes, + }, + "connections/:connectionId/scopes/:scopeId": { + "GET": api.GetScope, + "PATCH": api.PatchScope, + "DELETE": api.DeleteScope, + }, + "connections/:connectionId/scopes/:scopeId/latest-sync-state": { + "GET": api.GetScopeLatestSyncState, + }, + } +} + +func (p Incidentio) MakeDataSourcePipelinePlanV200( + connectionId uint64, + scopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes) +} + +func (p Incidentio) Close(taskCtx plugin.TaskContext) errors.Error { + _, ok := taskCtx.GetData().(*tasks.IncidentioTaskData) + if !ok { + return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) + } + return nil +} diff --git a/backend/plugins/incidentio/incidentio.go b/backend/plugins/incidentio/incidentio.go new file mode 100644 index 00000000000..e24d12d20f6 --- /dev/null +++ b/backend/plugins/incidentio/incidentio.go @@ -0,0 +1,38 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/incidentio/impl" + "github.com/spf13/cobra" +) + +// PluginEntry Export a variable named PluginEntry for Framework to search and load +var PluginEntry impl.Incidentio //nolint + +// standalone mode for debugging +func main() { + cmd := &cobra.Command{Use: "incidentio"} + timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z") + + cmd.Run = func(cmd *cobra.Command, args []string) { + runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{}, *timeAfter) + } + runner.RunCmd(cmd) +} diff --git a/backend/plugins/incidentio/models/connection.go b/backend/plugins/incidentio/models/connection.go new file mode 100644 index 00000000000..333bb509917 --- /dev/null +++ b/backend/plugins/incidentio/models/connection.go @@ -0,0 +1,74 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "fmt" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +type IncidentioAccessToken helper.AccessToken + +func (at *IncidentioAccessToken) SetupAuthentication(request *http.Request) errors.Error { + request.Header.Set("Authorization", fmt.Sprintf("Bearer %s", at.Token)) + return nil +} + +type IncidentioConn struct { + helper.RestConnection `mapstructure:",squash"` + IncidentioAccessToken `mapstructure:",squash"` +} + +func (connection IncidentioConn) Sanitize() IncidentioConn { + connection.Token = utils.SanitizeString(connection.Token) + return connection +} + +type IncidentioConnection struct { + helper.BaseConnection `mapstructure:",squash"` + IncidentioConn `mapstructure:",squash"` +} + +// MergeFromRequest preserves the existing token when an incoming PATCH +// body omits it or echoes the sanitized form. The config-UI sends the +// sanitized token back on every PATCH to avoid round-tripping the +// secret; this guard is what makes that pattern safe. +func (connection *IncidentioConnection) MergeFromRequest(target *IncidentioConnection, body map[string]interface{}) error { + token := target.Token + if err := helper.DecodeMapStruct(body, target, true); err != nil { + return err + } + modifiedToken := target.Token + if modifiedToken == "" || modifiedToken == utils.SanitizeString(token) { + target.Token = token + } + return nil +} + +func (IncidentioConnection) TableName() string { + return "_tool_incidentio_connections" +} + +func (connection IncidentioConnection) Sanitize() IncidentioConnection { + connection.Token = utils.SanitizeString(connection.Token) + return connection +} diff --git a/backend/plugins/incidentio/models/incident.go b/backend/plugins/incidentio/models/incident.go new file mode 100644 index 00000000000..7a8d42d2908 --- /dev/null +++ b/backend/plugins/incidentio/models/incident.go @@ -0,0 +1,46 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +type Incident struct { + common.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;autoIncrement:false"` + Reference string + Name string + Summary string + Url string + Mode string + StatusName string + StatusCategory string + SeverityName string + SeverityRank int64 + IncidentTypeId string `gorm:"index"` + CreatedDate time.Time + UpdatedDate time.Time + DeclaredDate time.Time + ResolvedDate *time.Time +} + +func (Incident) TableName() string { return "_tool_incidentio_incidents" } diff --git a/backend/plugins/incidentio/models/incident_type.go b/backend/plugins/incidentio/models/incident_type.go new file mode 100644 index 00000000000..3aa3075b8f4 --- /dev/null +++ b/backend/plugins/incidentio/models/incident_type.go @@ -0,0 +1,59 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" +) + +type IncidentioParams struct { + ConnectionId uint64 + ScopeId string +} + +type IncidentType struct { + common.Scope `mapstructure:",squash"` + Id string `json:"id" mapstructure:"id" gorm:"primaryKey;autoIncrement:false" ` + Name string `json:"name" mapstructure:"name"` +} + +func (t IncidentType) ScopeId() string { + return t.Id +} + +func (t IncidentType) ScopeName() string { + return t.Name +} + +func (t IncidentType) ScopeFullName() string { + return t.Name +} + +func (t IncidentType) ScopeParams() interface{} { + return &IncidentioParams{ + ConnectionId: t.ConnectionId, + ScopeId: t.Id, + } +} + +func (t IncidentType) TableName() string { + return "_tool_incidentio_incident_types" +} + +var _ plugin.ToolLayerScope = (*IncidentType)(nil) diff --git a/backend/plugins/incidentio/models/migrationscripts/20260729_add_init_tables.go b/backend/plugins/incidentio/models/migrationscripts/20260729_add_init_tables.go new file mode 100644 index 00000000000..fbdeb314d04 --- /dev/null +++ b/backend/plugins/incidentio/models/migrationscripts/20260729_add_init_tables.go @@ -0,0 +1,44 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/incidentio/models/migrationscripts/archived" +) + +type addInitTables struct{} + +func (*addInitTables) Up(baseRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(baseRes, + &archived.Connection{}, + &archived.IncidentType{}, + &archived.Incident{}, + &archived.ScopeConfig{}, + ) +} + +func (*addInitTables) Version() uint64 { + return 20260729000001 +} + +func (*addInitTables) Name() string { + return "incident.io init schemas" +} diff --git a/backend/plugins/incidentio/models/migrationscripts/archived/connection.go b/backend/plugins/incidentio/models/migrationscripts/archived/connection.go new file mode 100644 index 00000000000..7f176fc46b0 --- /dev/null +++ b/backend/plugins/incidentio/models/migrationscripts/archived/connection.go @@ -0,0 +1,35 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import ( + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +type Connection struct { + archived.Model + Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"` + Endpoint string `mapstructure:"endpoint" validate:"required" json:"endpoint"` + Proxy string `mapstructure:"proxy" json:"proxy"` + RateLimitPerHour int `comment:"api request rate limit per hour" json:"rateLimitPerHour"` + Token string `mapstructure:"token" env:"INCIDENTIO_AUTH" validate:"required" encrypt:"yes"` +} + +func (Connection) TableName() string { + return "_tool_incidentio_connections" +} diff --git a/backend/plugins/incidentio/models/migrationscripts/archived/incident.go b/backend/plugins/incidentio/models/migrationscripts/archived/incident.go new file mode 100644 index 00000000000..f665fec613a --- /dev/null +++ b/backend/plugins/incidentio/models/migrationscripts/archived/incident.go @@ -0,0 +1,48 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +type Incident struct { + archived.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;autoIncrement:false"` + Reference string + Name string + Summary string + Url string + Mode string + StatusName string + StatusCategory string + SeverityName string + SeverityRank int64 + IncidentTypeId string `gorm:"index"` + CreatedDate time.Time + UpdatedDate time.Time + DeclaredDate time.Time + ResolvedDate *time.Time +} + +func (Incident) TableName() string { + return "_tool_incidentio_incidents" +} diff --git a/backend/plugins/incidentio/models/migrationscripts/archived/incident_type.go b/backend/plugins/incidentio/models/migrationscripts/archived/incident_type.go new file mode 100644 index 00000000000..8dca3393fd4 --- /dev/null +++ b/backend/plugins/incidentio/models/migrationscripts/archived/incident_type.go @@ -0,0 +1,36 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import ( + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +// ScopeConfigId mirrors the column that live `models.IncidentType` gets +// via embedded `common.Scope`; the archived `NoPKModel` doesn't include it. +type IncidentType struct { + archived.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ScopeConfigId uint64 `json:"scopeConfigId,omitempty" mapstructure:"scopeConfigId,omitempty"` + Id string `gorm:"primaryKey;autoIncrement:false"` + Name string +} + +func (IncidentType) TableName() string { + return "_tool_incidentio_incident_types" +} diff --git a/backend/plugins/incidentio/models/migrationscripts/archived/scope_config.go b/backend/plugins/incidentio/models/migrationscripts/archived/scope_config.go new file mode 100644 index 00000000000..8b6174ef23b --- /dev/null +++ b/backend/plugins/incidentio/models/migrationscripts/archived/scope_config.go @@ -0,0 +1,35 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import ( + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +// ConnectionId and Name come from `common.ScopeConfig` on the live +// model; the archived `archived.ScopeConfig` base only carries +// Model + Entities, so declare them explicitly. +type ScopeConfig struct { + archived.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"` + ConnectionId uint64 `json:"connectionId" gorm:"index" validate:"required" mapstructure:"connectionId,omitempty"` + Name string `mapstructure:"name" json:"name" gorm:"type:varchar(255);uniqueIndex" validate:"required"` +} + +func (ScopeConfig) TableName() string { + return "_tool_incidentio_scope_configs" +} diff --git a/backend/plugins/incidentio/models/migrationscripts/register.go b/backend/plugins/incidentio/models/migrationscripts/register.go new file mode 100644 index 00000000000..16c72e29273 --- /dev/null +++ b/backend/plugins/incidentio/models/migrationscripts/register.go @@ -0,0 +1,29 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/plugin" +) + +// All returns all the migration scripts for the incidentio plugin +func All() []plugin.MigrationScript { + return []plugin.MigrationScript{ + new(addInitTables), + } +} diff --git a/backend/plugins/incidentio/models/raw/incident.go b/backend/plugins/incidentio/models/raw/incident.go new file mode 100644 index 00000000000..4bdde7ce38e --- /dev/null +++ b/backend/plugins/incidentio/models/raw/incident.go @@ -0,0 +1,78 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package raw + +import ( + "time" +) + +type Incident struct { + Id string `json:"id"` + Reference string `json:"reference"` + Name string `json:"name"` + Summary *string `json:"summary"` + Permalink *string `json:"permalink"` + Mode string `json:"mode"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + IncidentStatus *IncidentStatus `json:"incident_status"` + Severity *Severity `json:"severity"` + IncidentType *IncidentTypeRef `json:"incident_type"` + Creator *Creator `json:"creator"` + IncidentTimestampValues []IncidentTimestampValue `json:"incident_timestamp_values"` +} + +type IncidentStatus struct { + Name string `json:"name"` + Category string `json:"category"` +} + +type Severity struct { + Id string `json:"id"` + Name string `json:"name"` + Rank int64 `json:"rank"` +} + +type IncidentTypeRef struct { + Id string `json:"id"` + Name string `json:"name"` +} + +type Creator struct { + User *CreatorUser `json:"user"` +} + +type CreatorUser struct { + Id string `json:"id"` + Name string `json:"name"` + Email string `json:"email"` +} + +type IncidentTimestampValue struct { + IncidentTimestamp IncidentTimestamp `json:"incident_timestamp"` + Value *TimestampValue `json:"value"` +} + +type IncidentTimestamp struct { + Id string `json:"id"` + Name string `json:"name"` +} + +type TimestampValue struct { + Value time.Time `json:"value"` +} diff --git a/backend/plugins/incidentio/models/raw/incident_type.go b/backend/plugins/incidentio/models/raw/incident_type.go new file mode 100644 index 00000000000..baef2ca3404 --- /dev/null +++ b/backend/plugins/incidentio/models/raw/incident_type.go @@ -0,0 +1,28 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package raw + +import "time" + +type IncidentType struct { + Id string `json:"id"` + Name string `json:"name"` + Description *string `json:"description"` + CreatedAt *time.Time `json:"created_at"` + UpdatedAt *time.Time `json:"updated_at"` +} diff --git a/backend/plugins/incidentio/models/scope_config.go b/backend/plugins/incidentio/models/scope_config.go new file mode 100644 index 00000000000..f3220e1fa4f --- /dev/null +++ b/backend/plugins/incidentio/models/scope_config.go @@ -0,0 +1,30 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +type IncidentioScopeConfig struct { + common.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"` +} + +func (IncidentioScopeConfig) TableName() string { + return "_tool_incidentio_scope_configs" +} diff --git a/backend/plugins/incidentio/tasks/incident_type_converter.go b/backend/plugins/incidentio/tasks/incident_type_converter.go new file mode 100644 index 00000000000..6744b461068 --- /dev/null +++ b/backend/plugins/incidentio/tasks/incident_type_converter.go @@ -0,0 +1,81 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/incidentio/models" +) + +var ConvertIncidentTypesMeta = plugin.SubTaskMeta{ + Name: "convertIncidentTypes", + EntryPoint: ConvertIncidentTypes, + EnabledByDefault: true, + Description: "Convert incident.io incident types", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ConvertIncidentTypes(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*IncidentioTaskData) + rawDataSubTaskArgs := &helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Options: data.Options, + Table: RAW_INCIDENT_TYPES_TABLE, + } + clauses := []dal.Clause{ + dal.Select("incident_types.*"), + dal.From("_tool_incidentio_incident_types incident_types"), + dal.Where("id = ? and connection_id = ?", data.Options.IncidentTypeId, data.Options.ConnectionId), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: *rawDataSubTaskArgs, + InputRowType: reflect.TypeOf(models.IncidentType{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + incidentType := inputRow.(*models.IncidentType) + domainBoard := &ticket.Board{ + DomainEntity: domainlayer.DomainEntity{ + Id: didgen.NewDomainIdGenerator(incidentType).Generate(incidentType.ConnectionId, incidentType.Id), + }, + Name: incidentType.Name, + } + return []interface{}{ + domainBoard, + }, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/incidentio/tasks/incident_types_collector.go b/backend/plugins/incidentio/tasks/incident_types_collector.go new file mode 100644 index 00000000000..d440ed00923 --- /dev/null +++ b/backend/plugins/incidentio/tasks/incident_types_collector.go @@ -0,0 +1,72 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const RAW_INCIDENT_TYPES_TABLE = "incidentio_incident_types" + +// incidentTypesResponse is the envelope returned by +// GET /v1/incident_types. The endpoint is not paginated. +type incidentTypesResponse struct { + IncidentTypes []json.RawMessage `json:"incident_types"` +} + +var _ plugin.SubTaskEntryPoint = CollectIncidentTypes + +var CollectIncidentTypesMeta = plugin.SubTaskMeta{ + Name: "collectIncidentTypes", + EntryPoint: CollectIncidentTypes, + EnabledByDefault: true, + Description: "Collect incident.io incident types", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + ProductTables: []string{RAW_INCIDENT_TYPES_TABLE}, +} + +func CollectIncidentTypes(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*IncidentioTaskData) + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Options: data.Options, + Table: RAW_INCIDENT_TYPES_TABLE, + }, + ApiClient: data.Client, + UrlTemplate: "v1/incident_types", + Query: nil, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + rawResult := incidentTypesResponse{} + err := api.UnmarshalResponse(res, &rawResult) + if err != nil { + return nil, err + } + return rawResult.IncidentTypes, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/incidentio/tasks/incident_types_extractor.go b/backend/plugins/incidentio/tasks/incident_types_extractor.go new file mode 100644 index 00000000000..a5fb1bcb1fe --- /dev/null +++ b/backend/plugins/incidentio/tasks/incident_types_extractor.go @@ -0,0 +1,78 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/incidentio/models" + "github.com/apache/incubator-devlake/plugins/incidentio/models/raw" +) + +var _ plugin.SubTaskEntryPoint = ExtractIncidentTypes + +var ExtractIncidentTypesMeta = plugin.SubTaskMeta{ + Name: "extractIncidentTypes", + EntryPoint: ExtractIncidentTypes, + EnabledByDefault: true, + Description: "Extract incident.io incident types", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + ProductTables: []string{models.IncidentType{}.TableName()}, +} + +func ExtractIncidentTypes(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*IncidentioTaskData) + db := taskCtx.GetDal() + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Options: data.Options, + Table: RAW_INCIDENT_TYPES_TABLE, + }, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + rawIncidentType := &raw.IncidentType{} + if err := errors.Convert(json.Unmarshal(row.Data, rawIncidentType)); err != nil { + return nil, err + } + // The collector fetches the full incident type list; keep only + // the type this scope is bound to. + if data.Options.IncidentTypeId != "" && rawIncidentType.Id != data.Options.IncidentTypeId { + return nil, nil + } + incidentType := &models.IncidentType{ + Id: rawIncidentType.Id, + Name: rawIncidentType.Name, + } + incidentType.ConnectionId = data.Options.ConnectionId + // Preserve operator-set ScopeConfigId across re-collections. + existing := &models.IncidentType{} + if err := db.First(existing, dal.Where("connection_id = ? AND id = ?", data.Options.ConnectionId, rawIncidentType.Id)); err == nil { + incidentType.ScopeConfigId = existing.ScopeConfigId + } + return []interface{}{incidentType}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/incidentio/tasks/incidents_collector.go b/backend/plugins/incidentio/tasks/incidents_collector.go new file mode 100644 index 00000000000..de4f730e190 --- /dev/null +++ b/backend/plugins/incidentio/tasks/incidents_collector.go @@ -0,0 +1,129 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const RAW_INCIDENTS_TABLE = "incidentio_incidents" + +var _ plugin.SubTaskEntryPoint = CollectIncidents + +type collectedIncidents struct { + Incidents []json.RawMessage `json:"incidents"` + PaginationMeta *collectedPaginationMeta `json:"pagination_meta"` +} + +type collectedPaginationMeta struct { + After *string `json:"after"` + PageSize int `json:"page_size"` +} + +var CollectIncidentsMeta = plugin.SubTaskMeta{ + Name: "collectIncidents", + EntryPoint: CollectIncidents, + EnabledByDefault: true, + Description: "Collect incident.io incidents", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + ProductTables: []string{RAW_INCIDENTS_TABLE}, +} + +func CollectIncidents(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*IncidentioTaskData) + args := api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Options: data.Options, + Table: RAW_INCIDENTS_TABLE, + } + // Pagination state captured during ResponseParser and consulted in + // GetNextPageCustomData. Required because prevPageResponse.Body is + // a single-read stream and is already drained by the time the + // next-page hook fires. + var lastAfter *string + + collector, err := api.NewStatefulApiCollectorForFinalizableEntity(api.FinalizableApiCollectorArgs{ + RawDataSubTaskArgs: args, + ApiClient: data.Client, + CollectNewRecordsByList: api.FinalizableApiCollectorListArgs{ + PageSize: 250, + GetNextPageCustomData: func(prevReqData *api.RequestData, prevPageResponse *http.Response) (interface{}, errors.Error) { + // Safety cap against an upstream that returns full pages forever + // while echoing a non-empty `after` cursor on every page. + const maxPages = 10000 + if prevReqData.Pager.Page >= maxPages { + return nil, api.ErrFinishCollect + } + if lastAfter == nil || *lastAfter == "" { + return nil, api.ErrFinishCollect + } + return *lastAfter, nil + }, + FinalizableApiCollectorCommonArgs: api.FinalizableApiCollectorCommonArgs{ + UrlTemplate: "v2/incidents", + // incident.io does not support server-side filtering by incident + // type or update time on this endpoint, so every scope collects + // all incidents and the extractor filters; createdAfter is + // intentionally ignored. + Query: func(reqData *api.RequestData, createdAfter *time.Time) (url.Values, errors.Error) { + after := "" + if cursor, ok := reqData.CustomData.(string); ok { + after = cursor + } + return buildIncidentsQuery(reqData.Pager.Size, after), nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + rawResult := collectedIncidents{} + if err := api.UnmarshalResponse(res, &rawResult); err != nil { + return nil, err + } + if rawResult.PaginationMeta != nil { + lastAfter = rawResult.PaginationMeta.After + } else { + lastAfter = nil + } + return rawResult.Incidents, nil + }, + }, + }, + }) + if err != nil { + return err + } + return collector.Execute() +} + +// buildIncidentsQuery is the pure-function core of the Query closure +// above. incident.io paginates with an opaque `after` cursor taken from +// the previous response's pagination_meta; the first page sends no cursor. +func buildIncidentsQuery(pageSize int, after string) url.Values { + query := url.Values{} + query.Set("page_size", fmt.Sprintf("%d", pageSize)) + if after != "" { + query.Set("after", after) + } + return query +} diff --git a/backend/plugins/incidentio/tasks/incidents_collector_test.go b/backend/plugins/incidentio/tasks/incidents_collector_test.go new file mode 100644 index 00000000000..f9dea868412 --- /dev/null +++ b/backend/plugins/incidentio/tasks/incidents_collector_test.go @@ -0,0 +1,64 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBuildIncidentsQuery_FirstPage(t *testing.T) { + q := buildIncidentsQuery(250, "") + assert.Equal(t, "250", q.Get("page_size")) + assert.False(t, q.Has("after"), "first page must not send an after cursor") +} + +func TestBuildIncidentsQuery_SubsequentPage(t *testing.T) { + q := buildIncidentsQuery(250, "01FCNDV6P870EA6S7TK1DSYDG0") + assert.Equal(t, "250", q.Get("page_size")) + assert.Equal(t, "01FCNDV6P870EA6S7TK1DSYDG0", q.Get("after")) +} + +func TestCollectedIncidentsEnvelope(t *testing.T) { + body := []byte(`{ + "incidents": [{"id": "inc_1"}, {"id": "inc_2"}], + "pagination_meta": {"after": "cursor-abc", "page_size": 2} + }`) + rawResult := collectedIncidents{} + require.NoError(t, json.Unmarshal(body, &rawResult)) + require.Len(t, rawResult.Incidents, 2) + require.NotNil(t, rawResult.PaginationMeta) + require.NotNil(t, rawResult.PaginationMeta.After) + assert.Equal(t, "cursor-abc", *rawResult.PaginationMeta.After) +} + +func TestCollectedIncidentsEnvelope_LastPage(t *testing.T) { + // The final page omits `after` entirely. + body := []byte(`{ + "incidents": [{"id": "inc_3"}], + "pagination_meta": {"page_size": 250} + }`) + rawResult := collectedIncidents{} + require.NoError(t, json.Unmarshal(body, &rawResult)) + require.Len(t, rawResult.Incidents, 1) + require.NotNil(t, rawResult.PaginationMeta) + assert.Nil(t, rawResult.PaginationMeta.After) +} diff --git a/backend/plugins/incidentio/tasks/incidents_converter.go b/backend/plugins/incidentio/tasks/incidents_converter.go new file mode 100644 index 00000000000..90247c44d0c --- /dev/null +++ b/backend/plugins/incidentio/tasks/incidents_converter.go @@ -0,0 +1,153 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/incidentio/models" +) + +var _ plugin.SubTaskEntryPoint = ConvertIncidents + +var ConvertIncidentsMeta = plugin.SubTaskMeta{ + Name: "convertIncidents", + EntryPoint: ConvertIncidents, + EnabledByDefault: true, + Description: "Convert incident.io incidents into domain-layer ticket issues", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ConvertIncidents(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*IncidentioTaskData) + logger := taskCtx.GetLogger() + + cursor, err := db.Cursor( + dal.From(&models.Incident{}), + dal.Where("connection_id = ? AND incident_type_id = ?", data.Options.ConnectionId, data.Options.IncidentTypeId), + ) + if err != nil { + return err + } + defer cursor.Close() + + idGen := didgen.NewDomainIdGenerator(&models.Incident{}) + incidentTypeIdGen := didgen.NewDomainIdGenerator(&models.IncidentType{}) + boardId := incidentTypeIdGen.Generate(data.Options.ConnectionId, data.Options.IncidentTypeId) + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Options: data.Options, + Table: RAW_INCIDENTS_TABLE, + }, + InputRowType: reflect.TypeOf(models.Incident{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + incident := inputRow.(*models.Incident) + + status, known := mapStatusCategory(incident.StatusCategory) + if !known { + logger.Warn(nil, "unknown incident.io status category: %s", incident.StatusCategory) + } + + leadTime, resolutionDate := computeLeadTime(incident.DeclaredDate, incident.ResolvedDate) + + domainIssueId := idGen.Generate(data.Options.ConnectionId, incident.Id) + + domainIssue := &ticket.Issue{ + DomainEntity: domainlayer.DomainEntity{ + Id: domainIssueId, + }, + Url: incident.Url, + IssueKey: issueKeyFor(incident), + Title: incident.Name, + Description: incident.Summary, + Type: ticket.INCIDENT, + Status: status, + OriginalStatus: incident.StatusName, + ResolutionDate: resolutionDate, + CreatedDate: &incident.DeclaredDate, + UpdatedDate: &incident.UpdatedDate, + LeadTimeMinutes: leadTime, + Severity: incident.SeverityName, + } + + return []interface{}{ + domainIssue, + &ticket.BoardIssue{ + BoardId: boardId, + IssueId: domainIssueId, + }, + }, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} + +// Unknown categories fall through to IN_PROGRESS rather than +// panicking; incident.io statuses are operator-defined, but their +// categories form a small fixed enum, so anything new from upstream +// shouldn't crash a production pipeline. +func mapStatusCategory(category string) (mapped string, known bool) { + switch category { + case "triage", "declared", "active", "paused", "post-incident": + return ticket.IN_PROGRESS, true + case "closed", "resolved": + return ticket.DONE, true + default: + return ticket.IN_PROGRESS, false + } +} + +func computeLeadTime(declared time.Time, resolved *time.Time) (*uint, *time.Time) { + if resolved == nil { + return nil, nil + } + // Retrospective incidents are declared after the fact, so resolved + // legitimately precedes declared. The resolution date is still real; + // only the declared→resolved duration is meaningless (and a naive + // uint() cast on a negative duration would wrap to huge garbage and + // silently corrupt MTTR), so keep the date and drop the lead time. + if resolved.Before(declared) { + resolutionDate := *resolved + return nil, &resolutionDate + } + minutes := uint(resolved.Sub(declared).Minutes()) + resolutionDate := *resolved + return &minutes, &resolutionDate +} + +func issueKeyFor(incident *models.Incident) string { + if incident.Reference != "" { + return incident.Reference + } + return incident.Id +} diff --git a/backend/plugins/incidentio/tasks/incidents_converter_test.go b/backend/plugins/incidentio/tasks/incidents_converter_test.go new file mode 100644 index 00000000000..dc9924a93a9 --- /dev/null +++ b/backend/plugins/incidentio/tasks/incidents_converter_test.go @@ -0,0 +1,114 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/plugins/incidentio/models" +) + +func TestMapStatusCategory(t *testing.T) { + cases := []struct { + in string + expectMapped string + expectedKnown bool + }{ + {"triage", ticket.IN_PROGRESS, true}, + {"declared", ticket.IN_PROGRESS, true}, + {"active", ticket.IN_PROGRESS, true}, + {"paused", ticket.IN_PROGRESS, true}, + {"post-incident", ticket.IN_PROGRESS, true}, + {"closed", ticket.DONE, true}, + {"resolved", ticket.DONE, true}, + {"wat", ticket.IN_PROGRESS, false}, + {"", ticket.IN_PROGRESS, false}, + } + for _, c := range cases { + t.Run(c.in, func(t *testing.T) { + mapped, known := mapStatusCategory(c.in) + assert.Equal(t, c.expectMapped, mapped) + assert.Equal(t, c.expectedKnown, known) + }) + } +} + +func TestMapStatusCategoryDoesNotPanic(t *testing.T) { + assert.NotPanics(t, func() { + _, _ = mapStatusCategory("brand-new-category-incidentio-invented-yesterday") + }) +} + +func TestComputeLeadTime_Resolved(t *testing.T) { + declared := time.Date(2026, 5, 10, 10, 0, 0, 0, time.UTC) + resolved := time.Date(2026, 5, 10, 11, 30, 0, 0, time.UTC) + leadTime, resolutionDate := computeLeadTime(declared, &resolved) + require.NotNil(t, leadTime) + require.NotNil(t, resolutionDate) + assert.Equal(t, uint(90), *leadTime) + assert.Equal(t, resolved, *resolutionDate) +} + +func TestComputeLeadTime_Unresolved(t *testing.T) { + declared := time.Date(2026, 5, 10, 10, 0, 0, 0, time.UTC) + leadTime, resolutionDate := computeLeadTime(declared, nil) + assert.Nil(t, leadTime) + assert.Nil(t, resolutionDate) +} + +func TestComputeLeadTime_ZeroDuration(t *testing.T) { + declared := time.Date(2026, 5, 10, 10, 0, 0, 0, time.UTC) + resolved := declared + leadTime, resolutionDate := computeLeadTime(declared, &resolved) + require.NotNil(t, leadTime) + require.NotNil(t, resolutionDate) + assert.Equal(t, uint(0), *leadTime) +} + +// Retrospective incidents are declared after resolution: the resolution +// date must survive even though the declared→resolved duration is +// meaningless and the lead time is dropped. +func TestComputeLeadTime_ResolvedBeforeDeclared(t *testing.T) { + declared := time.Date(2026, 5, 10, 11, 0, 0, 0, time.UTC) + resolved := time.Date(2026, 5, 10, 10, 0, 0, 0, time.UTC) + leadTime, resolutionDate := computeLeadTime(declared, &resolved) + assert.Nil(t, leadTime) + require.NotNil(t, resolutionDate) + assert.Equal(t, resolved, *resolutionDate) +} + +func TestIssueKeyFor(t *testing.T) { + cases := []struct { + name string + incident models.Incident + expected string + }{ + {"reference present", models.Incident{Reference: "INC-61", Id: "inc_abc"}, "INC-61"}, + {"missing reference falls back to id", models.Incident{Reference: "", Id: "inc_abc"}, "inc_abc"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + assert.Equal(t, c.expected, issueKeyFor(&c.incident)) + }) + } +} diff --git a/backend/plugins/incidentio/tasks/incidents_extractor.go b/backend/plugins/incidentio/tasks/incidents_extractor.go new file mode 100644 index 00000000000..6bfd9bdbcff --- /dev/null +++ b/backend/plugins/incidentio/tasks/incidents_extractor.go @@ -0,0 +1,141 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/incidentio/models" + "github.com/apache/incubator-devlake/plugins/incidentio/models/raw" +) + +var _ plugin.SubTaskEntryPoint = ExtractIncidents + +var ExtractIncidentsMeta = plugin.SubTaskMeta{ + Name: "extractIncidents", + EntryPoint: ExtractIncidents, + EnabledByDefault: true, + Description: "Extract incident.io incidents", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + ProductTables: []string{models.Incident{}.TableName()}, +} + +func ExtractIncidents(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*IncidentioTaskData) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Options: data.Options, + Table: RAW_INCIDENTS_TABLE, + }, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + return extractIncidentioIncident(row.Data, data.Options) + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} + +func extractIncidentioIncident(rawData []byte, op *IncidentioOptions) ([]interface{}, errors.Error) { + rawIncident := &raw.Incident{} + if err := errors.Convert(json.Unmarshal(rawData, rawIncident)); err != nil { + return nil, err + } + + // "test" and "tutorial" incidents are practice data; keep only + // "standard" and "retrospective" (and future real modes). + if rawIncident.Mode == "test" || rawIncident.Mode == "tutorial" { + return nil, nil + } + + // The collector fetches all incidents (no server-side filtering), so + // the scope filter lives here. When IncidentTypeId is empty we are + // collecting all incidents globally, so skip this check. + if op.IncidentTypeId != "" { + if rawIncident.IncidentType == nil || rawIncident.IncidentType.Id != op.IncidentTypeId { + return nil, nil + } + } + + if rawIncident.CreatedAt.IsZero() { + return nil, errors.Default.New("incident.io incident missing created_at") + } + + declaredDate := rawIncident.CreatedAt + if declared := timestampValueByName(rawIncident.IncidentTimestampValues, "Declared at"); declared != nil { + declaredDate = *declared + } + resolvedDate := timestampValueByName(rawIncident.IncidentTimestampValues, "Resolved at") + if resolvedDate == nil { + resolvedDate = timestampValueByName(rawIncident.IncidentTimestampValues, "Closed at") + } + + incident := &models.Incident{ + ConnectionId: op.ConnectionId, + Id: rawIncident.Id, + Reference: rawIncident.Reference, + Name: rawIncident.Name, + Summary: resolve(rawIncident.Summary), + Url: resolve(rawIncident.Permalink), + Mode: rawIncident.Mode, + CreatedDate: rawIncident.CreatedAt, + UpdatedDate: rawIncident.UpdatedAt, + DeclaredDate: declaredDate, + ResolvedDate: resolvedDate, + } + if rawIncident.IncidentStatus != nil { + incident.StatusName = rawIncident.IncidentStatus.Name + incident.StatusCategory = rawIncident.IncidentStatus.Category + } + if rawIncident.Severity != nil { + incident.SeverityName = rawIncident.Severity.Name + incident.SeverityRank = rawIncident.Severity.Rank + } + if rawIncident.IncidentType != nil { + incident.IncidentTypeId = rawIncident.IncidentType.Id + } + + return []interface{}{incident}, nil +} + +func timestampValueByName(values []raw.IncidentTimestampValue, name string) *time.Time { + for _, v := range values { + if v.IncidentTimestamp.Name != name { + continue + } + if v.Value == nil || v.Value.Value.IsZero() { + return nil + } + value := v.Value.Value + return &value + } + return nil +} + +func resolve[T any](t *T) T { + if t == nil { + return *new(T) + } + return *t +} diff --git a/backend/plugins/incidentio/tasks/incidents_extractor_test.go b/backend/plugins/incidentio/tasks/incidents_extractor_test.go new file mode 100644 index 00000000000..0a0f08e8cca --- /dev/null +++ b/backend/plugins/incidentio/tasks/incidents_extractor_test.go @@ -0,0 +1,289 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/apache/incubator-devlake/plugins/incidentio/models" +) + +const baseHappyPathActive = `{ + "id": "01FCNDV6P870EA6S7TK1DSYDG0", + "reference": "INC-61", + "name": "db outage", + "summary": "replica lag blew past threshold", + "permalink": "https://app.incident.io/example/incidents/61", + "mode": "standard", + "created_at": "2026-05-10T09:55:00Z", + "updated_at": "2026-05-10T10:05:00Z", + "incident_status": {"name": "Investigating", "category": "active"}, + "severity": {"id": "sev-uuid-1", "name": "Major", "rank": 2}, + "incident_type": {"id": "type_01", "name": "Default"}, + "creator": {"user": {"id": "usr_100", "name": "Reporter One", "email": "reporter@example.com"}}, + "incident_timestamp_values": [ + {"incident_timestamp": {"id": "ts_01", "name": "Reported at"}, "value": {"value": "2026-05-10T09:55:00Z"}}, + {"incident_timestamp": {"id": "ts_02", "name": "Declared at"}, "value": {"value": "2026-05-10T10:00:00Z"}} + ] +}` + +func newTestOptions() *IncidentioOptions { + return &IncidentioOptions{ + ConnectionId: 7, + IncidentTypeId: "type_01", + } +} + +func TestExtractIncidentioIncident_HappyPathActive(t *testing.T) { + op := newTestOptions() + results, err := extractIncidentioIncident([]byte(baseHappyPathActive), op) + require.NoError(t, err) + require.Len(t, results, 1) + + incident, ok := results[0].(*models.Incident) + require.True(t, ok, "first result should be *models.Incident") + assert.Equal(t, uint64(7), incident.ConnectionId) + assert.Equal(t, "01FCNDV6P870EA6S7TK1DSYDG0", incident.Id) + assert.Equal(t, "INC-61", incident.Reference) + assert.Equal(t, "db outage", incident.Name) + assert.Equal(t, "replica lag blew past threshold", incident.Summary) + assert.Equal(t, "https://app.incident.io/example/incidents/61", incident.Url) + assert.Equal(t, "standard", incident.Mode) + assert.Equal(t, "Investigating", incident.StatusName) + assert.Equal(t, "active", incident.StatusCategory) + assert.Equal(t, "Major", incident.SeverityName) + assert.Equal(t, int64(2), incident.SeverityRank) + assert.Equal(t, "type_01", incident.IncidentTypeId) + assert.Equal(t, time.Date(2026, 5, 10, 9, 55, 0, 0, time.UTC), incident.CreatedDate) + assert.Equal(t, time.Date(2026, 5, 10, 10, 5, 0, 0, time.UTC), incident.UpdatedDate) + assert.Equal(t, time.Date(2026, 5, 10, 10, 0, 0, 0, time.UTC), incident.DeclaredDate) + assert.Nil(t, incident.ResolvedDate) +} + +func TestExtractIncidentioIncident_Resolved(t *testing.T) { + raw := []byte(`{ + "id": "inc_02", + "reference": "INC-62", + "name": "cache cleared", + "mode": "standard", + "created_at": "2026-05-09T07:55:00Z", + "updated_at": "2026-05-09T09:01:00Z", + "incident_status": {"name": "Closed", "category": "closed"}, + "severity": {"id": "sev-uuid-3", "name": "Minor", "rank": 1}, + "incident_type": {"id": "type_01", "name": "Default"}, + "incident_timestamp_values": [ + {"incident_timestamp": {"id": "ts_02", "name": "Declared at"}, "value": {"value": "2026-05-09T08:00:00Z"}}, + {"incident_timestamp": {"id": "ts_03", "name": "Fixed at"}, "value": {"value": "2026-05-09T08:30:00Z"}}, + {"incident_timestamp": {"id": "ts_04", "name": "Resolved at"}, "value": {"value": "2026-05-09T09:00:00Z"}}, + {"incident_timestamp": {"id": "ts_05", "name": "Closed at"}, "value": {"value": "2026-05-09T09:01:00Z"}} + ] + }`) + op := newTestOptions() + results, err := extractIncidentioIncident(raw, op) + require.NoError(t, err) + require.Len(t, results, 1) + + incident := results[0].(*models.Incident) + assert.Equal(t, "Closed", incident.StatusName) + assert.Equal(t, "closed", incident.StatusCategory) + assert.Equal(t, time.Date(2026, 5, 9, 8, 0, 0, 0, time.UTC), incident.DeclaredDate) + require.NotNil(t, incident.ResolvedDate) + assert.Equal(t, time.Date(2026, 5, 9, 9, 0, 0, 0, time.UTC), *incident.ResolvedDate) +} + +func TestExtractIncidentioIncident_ClosedAtFallback(t *testing.T) { + raw := []byte(`{ + "id": "inc_03", + "reference": "INC-63", + "name": "closed without resolved timestamp", + "mode": "standard", + "created_at": "2026-05-09T07:55:00Z", + "updated_at": "2026-05-09T09:01:00Z", + "incident_status": {"name": "Closed", "category": "closed"}, + "incident_type": {"id": "type_01", "name": "Default"}, + "incident_timestamp_values": [ + {"incident_timestamp": {"id": "ts_04", "name": "Resolved at"}, "value": null}, + {"incident_timestamp": {"id": "ts_05", "name": "Closed at"}, "value": {"value": "2026-05-09T09:01:00Z"}} + ] + }`) + op := newTestOptions() + results, err := extractIncidentioIncident(raw, op) + require.NoError(t, err) + require.Len(t, results, 1) + + incident := results[0].(*models.Incident) + require.NotNil(t, incident.ResolvedDate) + assert.Equal(t, time.Date(2026, 5, 9, 9, 1, 0, 0, time.UTC), *incident.ResolvedDate) +} + +func TestExtractIncidentioIncident_DeclaredAtFallsBackToCreatedAt(t *testing.T) { + raw := []byte(`{ + "id": "inc_04", + "reference": "INC-64", + "name": "no declared timestamp", + "mode": "standard", + "created_at": "2026-05-10T12:00:00Z", + "updated_at": "2026-05-10T12:05:00Z", + "incident_status": {"name": "Investigating", "category": "active"}, + "incident_type": {"id": "type_01", "name": "Default"} + }`) + op := newTestOptions() + results, err := extractIncidentioIncident(raw, op) + require.NoError(t, err) + require.Len(t, results, 1) + incident := results[0].(*models.Incident) + assert.Equal(t, time.Date(2026, 5, 10, 12, 0, 0, 0, time.UTC), incident.DeclaredDate) + assert.Nil(t, incident.ResolvedDate) +} + +func TestExtractIncidentioIncident_NullSeverity(t *testing.T) { + raw := []byte(`{ + "id": "inc_05", + "reference": "INC-65", + "name": "no sev yet", + "mode": "standard", + "created_at": "2026-05-10T14:00:00Z", + "updated_at": "2026-05-10T14:05:00Z", + "incident_status": {"name": "Triage", "category": "triage"}, + "severity": null, + "incident_type": {"id": "type_01", "name": "Default"} + }`) + op := newTestOptions() + results, err := extractIncidentioIncident(raw, op) + require.NoError(t, err) + require.Len(t, results, 1) + incident := results[0].(*models.Incident) + assert.Equal(t, "", incident.SeverityName) + assert.Equal(t, int64(0), incident.SeverityRank) +} + +func TestExtractIncidentioIncident_TestModeSkipped(t *testing.T) { + raw := []byte(`{ + "id": "inc_test", + "reference": "INC-66", + "name": "practice run", + "mode": "test", + "created_at": "2026-05-10T15:00:00Z", + "updated_at": "2026-05-10T15:05:00Z", + "incident_type": {"id": "type_01", "name": "Default"} + }`) + op := newTestOptions() + results, err := extractIncidentioIncident(raw, op) + require.NoError(t, err) + assert.Empty(t, results, "test-mode incident should produce no rows") +} + +func TestExtractIncidentioIncident_TutorialModeSkipped(t *testing.T) { + raw := []byte(`{ + "id": "inc_tutorial", + "reference": "INC-67", + "name": "onboarding walkthrough", + "mode": "tutorial", + "created_at": "2026-05-10T15:00:00Z", + "updated_at": "2026-05-10T15:05:00Z", + "incident_type": {"id": "type_01", "name": "Default"} + }`) + op := newTestOptions() + results, err := extractIncidentioIncident(raw, op) + require.NoError(t, err) + assert.Empty(t, results, "tutorial-mode incident should produce no rows") +} + +func TestExtractIncidentioIncident_RetrospectiveModeKept(t *testing.T) { + raw := []byte(`{ + "id": "inc_retro", + "reference": "INC-68", + "name": "backfilled incident", + "mode": "retrospective", + "created_at": "2026-05-10T16:00:00Z", + "updated_at": "2026-05-10T16:05:00Z", + "incident_status": {"name": "Closed", "category": "closed"}, + "incident_type": {"id": "type_01", "name": "Default"} + }`) + op := newTestOptions() + results, err := extractIncidentioIncident(raw, op) + require.NoError(t, err) + require.Len(t, results, 1) + incident := results[0].(*models.Incident) + assert.Equal(t, "retrospective", incident.Mode) +} + +func TestExtractIncidentioIncident_WrongIncidentTypeSkipped(t *testing.T) { + raw := []byte(`{ + "id": "inc_wrong_type", + "reference": "INC-69", + "name": "other type", + "mode": "standard", + "created_at": "2026-05-10T18:00:00Z", + "updated_at": "2026-05-10T18:05:00Z", + "incident_type": {"id": "type_99", "name": "Security"} + }`) + op := newTestOptions() + results, err := extractIncidentioIncident(raw, op) + require.NoError(t, err) + assert.Empty(t, results, "incident for unrelated incident type should produce no rows") +} + +func TestExtractIncidentioIncident_MissingIncidentTypeSkippedWhenScoped(t *testing.T) { + raw := []byte(`{ + "id": "inc_no_type", + "reference": "INC-70", + "name": "type omitted", + "mode": "standard", + "created_at": "2026-05-10T19:00:00Z", + "updated_at": "2026-05-10T19:05:00Z" + }`) + op := newTestOptions() + results, err := extractIncidentioIncident(raw, op) + require.NoError(t, err) + assert.Empty(t, results) +} + +func TestExtractIncidentioIncident_GlobalCollectionKeepsAllTypes(t *testing.T) { + raw := []byte(`{ + "id": "inc_any_type", + "reference": "INC-71", + "name": "any type", + "mode": "standard", + "created_at": "2026-05-10T20:00:00Z", + "updated_at": "2026-05-10T20:05:00Z", + "incident_type": {"id": "type_99", "name": "Security"} + }`) + op := &IncidentioOptions{ConnectionId: 7} + results, err := extractIncidentioIncident(raw, op) + require.NoError(t, err) + require.Len(t, results, 1) +} + +func TestExtractIncidentioIncident_MissingCreatedAtReturnsError(t *testing.T) { + raw := []byte(`{ + "id": "inc_bad", + "reference": "INC-72", + "name": "bad row", + "mode": "standard", + "updated_at": "2026-05-10T20:05:00Z", + "incident_type": {"id": "type_01", "name": "Default"} + }`) + op := newTestOptions() + _, err := extractIncidentioIncident(raw, op) + assert.Error(t, err) +} diff --git a/backend/plugins/incidentio/tasks/task_data.go b/backend/plugins/incidentio/tasks/task_data.go new file mode 100644 index 00000000000..9659d39ebc3 --- /dev/null +++ b/backend/plugins/incidentio/tasks/task_data.go @@ -0,0 +1,76 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/incidentio/models" +) + +type IncidentioOptions struct { + ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId,omitempty"` + IncidentTypeId string `json:"incidentTypeId,omitempty" mapstructure:"incidentTypeId,omitempty"` + IncidentTypeName string `json:"incidentTypeName,omitempty" mapstructure:"incidentTypeName,omitempty"` + ScopeConfigId uint64 `json:"scopeConfigId,omitempty" mapstructure:"scopeConfigId,omitempty"` + ScopeConfig *models.IncidentioScopeConfig `json:"scopeConfig,omitempty" mapstructure:"scopeConfig,omitempty"` +} + +type IncidentioTaskData struct { + Options *IncidentioOptions + Client api.RateLimitedApiClient +} + +func (p *IncidentioOptions) GetParams() any { + scopeId := p.IncidentTypeId + if scopeId == "" { + scopeId = "all" + } + return models.IncidentioParams{ + ConnectionId: p.ConnectionId, + ScopeId: scopeId, + } +} + +func DecodeAndValidateTaskOptions(options map[string]interface{}) (*IncidentioOptions, errors.Error) { + op, err := DecodeTaskOptions(options) + if err != nil { + return nil, err + } + err = ValidateTaskOptions(op) + if err != nil { + return nil, err + } + return op, nil +} + +func DecodeTaskOptions(options map[string]interface{}) (*IncidentioOptions, errors.Error) { + var op IncidentioOptions + err := api.Decode(options, &op, nil) + if err != nil { + return nil, err + } + return &op, nil +} + +func ValidateTaskOptions(op *IncidentioOptions) errors.Error { + if op.ConnectionId == 0 { + return errors.BadInput.New("connectionId is invalid") + } + return nil +} diff --git a/backend/plugins/jira/api/connection_api.go b/backend/plugins/jira/api/connection_api.go index 9b6071b15fc..209ab6c5c99 100644 --- a/backend/plugins/jira/api/connection_api.go +++ b/backend/plugins/jira/api/connection_api.go @@ -58,7 +58,10 @@ func testConnection(ctx context.Context, connection models.JiraConn) (*JiraTestC } serverInfoFail := "Failed testing the serverInfo: [ " + res.Request.URL.String() + " ]" // check if `/rest/` was missing - if res.StatusCode == http.StatusNotFound && !strings.HasSuffix(connection.Endpoint, "/rest/") { + // Skip this hint for Atlassian API Gateway endpoints — they already include /rest/ in the path + // and a 404 on serverInfo there indicates a wrong Cloud ID, not a missing /rest/ segment. + isGatewayEndpoint := strings.Contains(connection.Endpoint, "api.atlassian.com") + if res.StatusCode == http.StatusNotFound && !strings.HasSuffix(connection.Endpoint, "/rest/") && !isGatewayEndpoint { endpointUrl, err := url.Parse(connection.Endpoint) if err != nil { return nil, errors.Convert(err) diff --git a/backend/plugins/jira/api/connection_api_test.go b/backend/plugins/jira/api/connection_api_test.go new file mode 100644 index 00000000000..e8ce6c0ad22 --- /dev/null +++ b/backend/plugins/jira/api/connection_api_test.go @@ -0,0 +1,144 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/apache/incubator-devlake/core/config" + implcontext "github.com/apache/incubator-devlake/impls/context" + "github.com/apache/incubator-devlake/impls/logruslog" + "github.com/apache/incubator-devlake/plugins/jira/models" + "github.com/go-playground/validator/v10" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// initTestDeps wires up the package-level basicRes and vld. +// DefaultBasicRes satisfies context.BasicRes and only needs config + logger; +// dal can be nil because NewApiClientFromConnection never calls GetDal. +func initTestDeps(t *testing.T) { + t.Helper() + basicRes = implcontext.NewDefaultBasicRes(config.GetConfig(), logruslog.Global, nil) + vld = validator.New() +} + +// serverInfoResponse is the minimal Jira serverInfo payload used in tests. +type serverInfoResponse struct { + DeploymentType string `json:"deploymentType"` + Version string `json:"version"` + VersionNumbers []int `json:"versionNumbers"` +} + +// newJiraTestServer creates an httptest.Server that responds to Jira REST paths. +// It records the Authorization header sent by the client for later assertion. +func newJiraTestServer(t *testing.T, authHeaderOut *string) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if authHeaderOut != nil { + *authHeaderOut = r.Header.Get("Authorization") + } + w.Header().Set("Content-Type", "application/json") + switch { + case strings.HasSuffix(r.URL.Path, "/api/2/serverInfo"): + json.NewEncoder(w).Encode(serverInfoResponse{ + DeploymentType: "Cloud", + Version: "1001.0.0", + VersionNumbers: []int{1001, 0, 0}, + }) + case strings.Contains(r.URL.Path, "/agile/1.0/board"): + json.NewEncoder(w).Encode(map[string]interface{}{"values": []interface{}{}}) + default: + http.NotFound(w, r) + } + })) +} + +// TestTestConnection_Gateway verifies that a gateway-style endpoint with +// authMethod=AccessToken sends "Authorization: Bearer " and succeeds. +func TestTestConnection_Gateway(t *testing.T) { + initTestDeps(t) + + var capturedAuth string + srv := newJiraTestServer(t, &capturedAuth) + defer srv.Close() + + conn := models.JiraConn{} + conn.Endpoint = srv.URL + "/rest/" + conn.AuthMethod = "AccessToken" + conn.Token = "my-scoped-gateway-token" + + resp, err := testConnection(context.Background(), conn) + require.Nil(t, err, "testConnection should succeed for gateway endpoint") + require.NotNil(t, resp) + assert.True(t, resp.Success) + assert.Equal(t, "Bearer my-scoped-gateway-token", capturedAuth, + "gateway mode must send Bearer token, not Basic credentials") +} + +// TestTestConnection_StandardCloud verifies backward compatibility: +// a standard atlassian.net endpoint with BasicAuth still works. +func TestTestConnection_StandardCloud(t *testing.T) { + initTestDeps(t) + + var capturedAuth string + srv := newJiraTestServer(t, &capturedAuth) + defer srv.Close() + + conn := models.JiraConn{} + conn.Endpoint = srv.URL + "/rest/" + conn.AuthMethod = "BasicAuth" + conn.Username = "user@example.com" + conn.Password = "api-token-value" + + resp, err := testConnection(context.Background(), conn) + require.Nil(t, err, "testConnection should succeed for standard cloud endpoint") + require.NotNil(t, resp) + assert.True(t, resp.Success) + assert.True(t, strings.HasPrefix(capturedAuth, "Basic "), + "standard cloud mode must send Basic auth header") +} + +// TestTestConnection_NonGateway404ShowsHint verifies that when a non-gateway endpoint +// is missing /rest/ and returns 404, the helpful hint message is included in the error. +func TestTestConnection_NonGateway404ShowsHint(t *testing.T) { + initTestDeps(t) + + // Server that always returns 404 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.NotFound(w, r) + })) + defer srv.Close() + + // URL intentionally missing /rest/ — should trigger the "please try" hint + connMissingRest := models.JiraConn{} + connMissingRest.Endpoint = srv.URL + "/" + connMissingRest.AuthMethod = "BasicAuth" + connMissingRest.Username = "u" + connMissingRest.Password = "p" + + _, errMissingRest := testConnection(context.Background(), connMissingRest) + require.NotNil(t, errMissingRest) + assert.Contains(t, errMissingRest.Error(), "please try", + "non-gateway 404 should include the /rest/ hint") +} diff --git a/backend/plugins/jira/e2e/migration_schema_test.go b/backend/plugins/jira/e2e/migration_schema_test.go new file mode 100644 index 00000000000..619aee6e415 --- /dev/null +++ b/backend/plugins/jira/e2e/migration_schema_test.go @@ -0,0 +1,110 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "sync" + "testing" + + "github.com/apache/incubator-devlake/core/config" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/migration" + coreMigration "github.com/apache/incubator-devlake/core/models/migrationscripts" + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/impls/dalgorm" + "github.com/apache/incubator-devlake/impls/logruslog" + "github.com/apache/incubator-devlake/plugins/jira/impl" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm/schema" +) + +// TestMigrationSchema guards against schema drift between Jira's migration +// scripts and its runtime GORM models. +// +// Regression test for the Sprint Report bug (introduced by PR #8967/#9010): +// the migration that created `_tool_jira_sprint_reports` used a struct that did +// NOT embed common.NoPKModel, so the `_raw_data_table` / `_raw_data_params` / +// `_raw_data_id` / `_raw_data_remark` columns were missing. The runtime model +// DID embed common.NoPKModel, so the ApiExtractor's cleanup query +// (`WHERE _raw_data_table = ? AND _raw_data_params = ?`) failed at runtime with +// "Error 1054: Unknown column '_raw_data_table' in 'where clause'". +// +// The test runs the REAL migration scripts (framework + jira) to build the +// schema exactly the way a production install would — deliberately NOT via +// AutoMigrate on the runtime model, which would silently hide such drift — and +// then asserts that every column each runtime model expects actually exists in +// the migrated table. Any future migration that forgets to embed +// common.NoPKModel (or otherwise omits a column) will fail this test. +// +// The migrations run against a dedicated, empty database (see +// e2ehelper.NewIsolatedMigrationDb) because the shared e2e database is polluted +// by the other e2e tests, which AutoMigrate tables without recording anything +// in `_devlake_migration_history`. +// +// Requires E2E_DB_URL (runs under `make e2e-test` / `make e2e-test-go-plugins`). +func TestMigrationSchema(t *testing.T) { + var pluginInstance impl.Jira + + db := e2ehelper.NewIsolatedMigrationDb(t, "jira_migration_schema") + dalInstance := dalgorm.NewDalgorm(db) + + // Apply the real migration scripts so the schema matches a production install. + basicRes := runner.CreateBasicRes(config.GetConfig(), logruslog.Global, db) + migrator, err := migration.NewMigrator(basicRes) + require.NoError(t, err) + migrator.Register(coreMigration.All(), "Framework") + migrator.Register(pluginInstance.MigrationScripts(), "jira") + require.NoError(t, migrator.Execute()) + + keepAll := func(dal.ColumnMeta) bool { return true } + + for _, table := range pluginInstance.GetTablesInfo() { + table := table + t.Run(table.TableName(), func(t *testing.T) { + // Columns the runtime GORM model expects. + sch, err := schema.Parse(table, &sync.Map{}, schema.NamingStrategy{}) + require.NoErrorf(t, err, "unable to parse schema for %T", table) + + // Columns that actually exist in the migrated table. + actualColumns, err := dal.GetColumnNames(dalInstance, table, keepAll) + if err != nil || len(actualColumns) == 0 { + // No migration creates this table (e.g. API response models + // that are listed in GetTablesInfo but never persisted) — + // there is no schema to drift from. + t.Skipf("table %q not present after migrations", table.TableName()) + } + existing := make(map[string]struct{}, len(actualColumns)) + for _, c := range actualColumns { + existing[c] = struct{}{} + } + + for _, field := range sch.Fields { + if field.DBName == "" || field.IgnoreMigration { + continue + } + _, ok := existing[field.DBName] + assert.Truef(t, ok, + "table %q is missing column %q expected by model %T — "+ + "did a migration script forget to embed common.NoPKModel (raw-data columns) or add the field?", + table.TableName(), field.DBName, table) + } + }) + } +} diff --git a/backend/plugins/jira/impl/impl.go b/backend/plugins/jira/impl/impl.go index 126e513e24d..b27ea9daa66 100644 --- a/backend/plugins/jira/impl/impl.go +++ b/backend/plugins/jira/impl/impl.go @@ -85,6 +85,7 @@ func (p Jira) GetTablesInfo() []dal.Tabler { &models.JiraServerInfo{}, &models.JiraSprint{}, &models.JiraSprintIssue{}, + &models.JiraSprintReport{}, &models.JiraStatus{}, &models.JiraWorklog{}, &models.JiraIssueComment{}, @@ -138,6 +139,9 @@ func (p Jira) SubTaskMetas() []plugin.SubTaskMeta { tasks.CollectSprintsMeta, tasks.ExtractSprintsMeta, + tasks.CollectSprintReportMeta, + tasks.ExtractSprintReportMeta, + tasks.CollectEpicsMeta, tasks.ExtractEpicsMeta, @@ -153,6 +157,7 @@ func (p Jira) SubTaskMetas() []plugin.SubTaskMeta { tasks.ConvertSprintsMeta, tasks.ConvertSprintIssuesMeta, + tasks.ConvertSprintReportMeta, tasks.CollectDevelopmentPanelMeta, tasks.ExtractDevelopmentPanelMeta, @@ -195,8 +200,8 @@ func (p Jira) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]int return nil, errors.Default.Wrap(err, "failed to create jira api client") } + var scope *models.JiraBoard if op.BoardId != 0 { - var scope *models.JiraBoard // support v100 & advance mode // If we still cannot find the record in db, we have to request from remote server and save it to db db := taskCtx.GetDal() @@ -249,6 +254,7 @@ func (p Jira) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]int Options: &op, ApiClient: jiraApiClient, JiraServerInfo: *info, + Board: scope, } return taskData, nil diff --git a/backend/plugins/jira/models/board.go b/backend/plugins/jira/models/board.go index 267c1cdc0ec..f76af2a5247 100644 --- a/backend/plugins/jira/models/board.go +++ b/backend/plugins/jira/models/board.go @@ -34,6 +34,7 @@ type JiraBoard struct { Self string `json:"self" mapstructure:"self" gorm:"type:varchar(255)"` Type string `json:"type" mapstructure:"type" gorm:"type:varchar(100)"` Jql string `json:"jql" mapstructure:"jql"` + SubQuery string `json:"subQuery" mapstructure:"subQuery"` } func (b JiraBoard) ScopeId() string { diff --git a/backend/plugins/jira/models/migrationscripts/20260611_add_sub_query_to_boards.go b/backend/plugins/jira/models/migrationscripts/20260611_add_sub_query_to_boards.go new file mode 100644 index 00000000000..455be7f6df2 --- /dev/null +++ b/backend/plugins/jira/models/migrationscripts/20260611_add_sub_query_to_boards.go @@ -0,0 +1,46 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type JiraBoard20260611 struct { + SubQuery string +} + +func (JiraBoard20260611) TableName() string { + return "_tool_jira_boards" +} + +type addSubQueryToBoards struct{} + +func (script *addSubQueryToBoards) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, &JiraBoard20260611{}) +} + +func (*addSubQueryToBoards) Version() uint64 { + return 20260611140000 +} + +func (*addSubQueryToBoards) Name() string { + return "add sub_query to _tool_jira_boards" +} diff --git a/backend/plugins/jira/models/migrationscripts/20260702_add_extra_jql_to_scope_config.go b/backend/plugins/jira/models/migrationscripts/20260702_add_extra_jql_to_scope_config.go new file mode 100644 index 00000000000..34966a56525 --- /dev/null +++ b/backend/plugins/jira/models/migrationscripts/20260702_add_extra_jql_to_scope_config.go @@ -0,0 +1,46 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type JiraScopeConfig20260702 struct { + ExtraJQL string `gorm:"type:varchar(255)"` +} + +func (JiraScopeConfig20260702) TableName() string { + return "_tool_jira_scope_configs" +} + +type addExtraJQLToScopeConfig struct{} + +func (script *addExtraJQLToScopeConfig) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, &JiraScopeConfig20260702{}) +} + +func (*addExtraJQLToScopeConfig) Version() uint64 { + return 20260702000000 +} + +func (*addExtraJQLToScopeConfig) Name() string { + return "add extra_jql to _tool_jira_scope_configs" +} diff --git a/backend/plugins/jira/models/migrationscripts/20260707_change_fix_versions_to_text.go b/backend/plugins/jira/models/migrationscripts/20260707_change_fix_versions_to_text.go new file mode 100644 index 00000000000..23b37dab081 --- /dev/null +++ b/backend/plugins/jira/models/migrationscripts/20260707_change_fix_versions_to_text.go @@ -0,0 +1,46 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type jiraIssue20260707 struct { + FixVersions string `gorm:"type:text;column:fix_versions"` +} + +func (jiraIssue20260707) TableName() string { + return "_tool_jira_issues" +} + +type changeFixVersionsToText20260707 struct{} + +func (script *changeFixVersionsToText20260707) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, &jiraIssue20260707{}) +} + +func (*changeFixVersionsToText20260707) Version() uint64 { + return 20260707140000 +} + +func (*changeFixVersionsToText20260707) Name() string { + return "change fix_versions type to text in _tool_jira_issues" +} diff --git a/backend/plugins/jira/models/migrationscripts/20260722_add_sprint_report_table.go b/backend/plugins/jira/models/migrationscripts/20260722_add_sprint_report_table.go new file mode 100644 index 00000000000..c31b709ded6 --- /dev/null +++ b/backend/plugins/jira/models/migrationscripts/20260722_add_sprint_report_table.go @@ -0,0 +1,55 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type jiraSprintReport20260722 struct { + ConnectionId uint64 `gorm:"primaryKey"` + BoardId uint64 `gorm:"primaryKey"` + SprintId uint64 `gorm:"primaryKey"` + IssueId uint64 `gorm:"primaryKey"` + + IssueKey string `gorm:"type:varchar(255)"` + Bucket string `gorm:"type:varchar(32);index"` + Done bool + StoryPointsAtSprintStart *float64 + StoryPointsAtSprintEnd *float64 +} + +func (jiraSprintReport20260722) TableName() string { + return "_tool_jira_sprint_reports" +} + +type addSprintReportTable struct{} + +func (script *addSprintReportTable) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, &jiraSprintReport20260722{}) +} + +func (*addSprintReportTable) Version() uint64 { + return 20260722000000 +} + +func (*addSprintReportTable) Name() string { + return "add _tool_jira_sprint_reports table to persist Jira's frozen Sprint Report snapshot" +} diff --git a/backend/plugins/jira/models/migrationscripts/20260727_add_raw_data_columns_to_sprint_report.go b/backend/plugins/jira/models/migrationscripts/20260727_add_raw_data_columns_to_sprint_report.go new file mode 100644 index 00000000000..5ee4d671f70 --- /dev/null +++ b/backend/plugins/jira/models/migrationscripts/20260727_add_raw_data_columns_to_sprint_report.go @@ -0,0 +1,65 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +// jiraSprintReport20260727 mirrors the JiraSprintReport model. The original +// migration (20260722) created _tool_jira_sprint_reports without embedding +// common.NoPKModel, so the _raw_data_table / _raw_data_params / _raw_data_id / +// _raw_data_remark columns (and created_at / updated_at) were missing. The +// runtime model expects them, which made the ApiExtractor's cleanup query +// (WHERE _raw_data_table = ? AND _raw_data_params = ?) fail with +// "Unknown column '_raw_data_table' in 'where clause'". Re-running +// AutoMigrateTables adds the missing columns without dropping existing data. +type jiraSprintReport20260727 struct { + archived.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + BoardId uint64 `gorm:"primaryKey"` + SprintId uint64 `gorm:"primaryKey"` + IssueId uint64 `gorm:"primaryKey"` + + IssueKey string `gorm:"type:varchar(255)"` + Bucket string `gorm:"type:varchar(32);index"` + Done bool + StoryPointsAtSprintStart *float64 + StoryPointsAtSprintEnd *float64 +} + +func (jiraSprintReport20260727) TableName() string { + return "_tool_jira_sprint_reports" +} + +type addRawDataColumnsToSprintReport struct{} + +func (script *addRawDataColumnsToSprintReport) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, &jiraSprintReport20260727{}) +} + +func (*addRawDataColumnsToSprintReport) Version() uint64 { + return 20260727000000 +} + +func (*addRawDataColumnsToSprintReport) Name() string { + return "add missing _raw_data_* columns to _tool_jira_sprint_reports" +} diff --git a/backend/plugins/jira/models/migrationscripts/register.go b/backend/plugins/jira/models/migrationscripts/register.go index 9c334a9ef88..e71dee8ea14 100644 --- a/backend/plugins/jira/models/migrationscripts/register.go +++ b/backend/plugins/jira/models/migrationscripts/register.go @@ -55,5 +55,10 @@ func All() []plugin.MigrationScript { new(flushJiraIssues), new(updateScopeConfig), new(addFixVersions20250619), + new(addSubQueryToBoards), + new(changeFixVersionsToText20260707), + new(addExtraJQLToScopeConfig), + new(addSprintReportTable), + new(addRawDataColumnsToSprintReport), } } diff --git a/backend/plugins/jira/models/scope_config.go b/backend/plugins/jira/models/scope_config.go index a8bd78a981b..5dc6d07fed9 100644 --- a/backend/plugins/jira/models/scope_config.go +++ b/backend/plugins/jira/models/scope_config.go @@ -19,6 +19,7 @@ package models import ( "regexp" + "text/template" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/common" @@ -49,6 +50,7 @@ type JiraScopeConfig struct { TypeMappings map[string]TypeMapping `mapstructure:"typeMappings,omitempty" json:"typeMappings" gorm:"type:json;serializer:json"` ApplicationType string `mapstructure:"applicationType,omitempty" json:"applicationType" gorm:"type:varchar(255)"` DueDateField string `mapstructure:"dueDateField,omitempty" json:"dueDateField" gorm:"type:varchar(255)"` + ExtraJQL string `mapstructure:"extraJql,omitempty" json:"extraJql" gorm:"type:varchar(255)"` } func (r *JiraScopeConfig) SetConnectionId(c *JiraScopeConfig, connectionId uint64) { @@ -73,6 +75,11 @@ func (r *JiraScopeConfig) Validate() errors.Error { return errors.Convert(err) } } + if r.ExtraJQL != "" { + if _, tmplErr := template.New("extraJql").Funcs(template.FuncMap{}).Option("missingkey=error").Parse(r.ExtraJQL); tmplErr != nil { + return errors.BadInput.Wrap(errors.Convert(tmplErr), "invalid ExtraJQL template") + } + } return nil } diff --git a/backend/plugins/jira/models/sprint_report.go b/backend/plugins/jira/models/sprint_report.go new file mode 100644 index 00000000000..b9280e2e41a --- /dev/null +++ b/backend/plugins/jira/models/sprint_report.go @@ -0,0 +1,64 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// Sprint Report bucket values, mirroring the four buckets returned by +// GET rest/greenhopper/1.0/rapid/charts/sprintreport. Jira computes these +// once, at sprint close, so persisting them (rather than reconstructing +// from resolution_date) is what makes committed/completed velocity exact. +const ( + SprintReportBucketCompleted = "completed" + SprintReportBucketNotCompleted = "notCompleted" + SprintReportBucketPunted = "punted" + SprintReportBucketCompletedInOtherSprint = "completedInOtherSprint" +) + +// JiraSprintReport is a frozen, per-(board, sprint, issue) snapshot taken +// from Jira's Sprint Report at sprint close. Unlike JiraSprintIssue (which +// is derived from each issue's live resolution_date and therefore +// mis-attributes carryover issues), this table stores Jira's own +// point-in-time bucketing, so it doesn't drift. +type JiraSprintReport struct { + common.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + BoardId uint64 `gorm:"primaryKey"` + SprintId uint64 `gorm:"primaryKey"` + IssueId uint64 `gorm:"primaryKey"` + + IssueKey string `gorm:"type:varchar(255)"` + // Bucket is one of the SprintReportBucket* constants above. + Bucket string `gorm:"type:varchar(32);index"` + Done bool + + // StoryPointsAtSprintStart is estimateStatistic.statFieldValue.value in + // Jira's response ("BOS points") — the estimate as it stood when the + // sprint began, i.e. what should be summed for *committed* velocity. + StoryPointsAtSprintStart *float64 + // StoryPointsAtSprintEnd is currentEstimateStatistic.statFieldValue.value + // ("EOS points") — the estimate as of sprint close, i.e. what should be + // summed (for Bucket == completed) for *completed* velocity. + StoryPointsAtSprintEnd *float64 +} + +func (JiraSprintReport) TableName() string { + return "_tool_jira_sprint_reports" +} diff --git a/backend/plugins/jira/tasks/apiv2models/sprint_report.go b/backend/plugins/jira/tasks/apiv2models/sprint_report.go new file mode 100644 index 00000000000..eec40ace9b9 --- /dev/null +++ b/backend/plugins/jira/tasks/apiv2models/sprint_report.go @@ -0,0 +1,73 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiv2models + +import "time" + +// SprintReportInput drives the per-(board, sprint) Sprint Report collector. +// It's what gets iterated over via the DAL cursor, and re-attached to each +// raw row so the extractor knows which board/sprint a response belongs to. +type SprintReportInput struct { + BoardId uint64 `json:"board_id"` + SprintId uint64 `json:"sprint_id"` + // UpdateTime is the sprint's CompleteDate; used as the incremental-sync + // watermark since a sprint report only exists/changes once a sprint closes. + UpdateTime *time.Time `json:"update_time"` +} + +// SprintReportStatFieldValue mirrors Jira's +// {statFieldValue: {value, text}} shape used for per-issue point estimates. +type SprintReportStatFieldValue struct { + Value *float64 `json:"value"` + Text string `json:"text"` +} + +type SprintReportStatistic struct { + StatFieldValue SprintReportStatFieldValue `json:"statFieldValue"` +} + +// SprintReportIssue is one entry inside any of the four bucket lists +// (completedIssues, issuesNotCompletedInCurrentSprint, puntedIssues, +// issuesCompletedInAnotherSprint) in the Sprint Report response. +type SprintReportIssue struct { + Id uint64 `json:"id"` + Key string `json:"key"` + TypeName string `json:"typeName"` + Done bool `json:"done"` + // EstimateStatistic is the issue's estimate as of sprint *start* + // ("BOS points" / committed). + EstimateStatistic SprintReportStatistic `json:"estimateStatistic"` + // CurrentEstimateStatistic is the issue's estimate as of sprint *close* + // ("EOS points" / completed). + CurrentEstimateStatistic SprintReportStatistic `json:"currentEstimateStatistic"` +} + +// SprintReportContents is the "contents" object of the Sprint Report +// response — the frozen snapshot Jira takes at sprint close. +type SprintReportContents struct { + CompletedIssues []SprintReportIssue `json:"completedIssues"` + IssuesNotCompletedInCurrentSprint []SprintReportIssue `json:"issuesNotCompletedInCurrentSprint"` + PuntedIssues []SprintReportIssue `json:"puntedIssues"` + IssuesCompletedInAnotherSprint []SprintReportIssue `json:"issuesCompletedInAnotherSprint"` +} + +// SprintReport is the top-level response of +// GET rest/greenhopper/1.0/rapid/charts/sprintreport?rapidViewId=&sprintId= +type SprintReport struct { + Contents SprintReportContents `json:"contents"` +} diff --git a/backend/plugins/jira/tasks/board_filter_begin_collector.go b/backend/plugins/jira/tasks/board_filter_begin_collector.go index 6c513f1f83c..10dc60758a8 100644 --- a/backend/plugins/jira/tasks/board_filter_begin_collector.go +++ b/backend/plugins/jira/tasks/board_filter_begin_collector.go @@ -41,14 +41,18 @@ func CollectBoardFilterBegin(taskCtx plugin.SubTaskContext) errors.Error { logger := taskCtx.GetLogger() db := taskCtx.GetDal() logger.Info("collect board in collectBoardFilterBegin: %d", data.Options.BoardId) - // get board filter id - filterId, err := getBoardFilterId(data) + + boardConfig, err := getBoardConfiguration(data) if err != nil { - return errors.Default.Wrap(err, fmt.Sprintf("error getting board filter id for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) + return errors.Default.Wrap(err, fmt.Sprintf("error getting board configuration for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) } + filterId := boardConfig.Filter.ID logger.Info("collect board filter:%s", filterId) - // get board filter jql + if boardConfig.SubQuery.Query != "" { + logger.Warn(nil, "board %d has kanban sub-filter: %s — using saved filter JQL for collection to avoid silent issue exclusion", data.Options.BoardId, boardConfig.SubQuery.Query) + } + filterInfo, err := getBoardFilterJql(data, filterId) if err != nil { return errors.Default.Wrap(err, fmt.Sprintf("error getting board filter jql for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) @@ -62,17 +66,21 @@ func CollectBoardFilterBegin(taskCtx plugin.SubTaskContext) errors.Error { return errors.Default.Wrap(err, fmt.Sprintf("error finding record in _tool_jira_boards table for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) } + // Store filter ID and sub-query on task data for downstream subtasks + data.FilterId = filterId + record.SubQuery = boardConfig.SubQuery.Query + // full sync syncPolicy := taskCtx.TaskContext().SyncPolicy() if syncPolicy != nil && syncPolicy.FullSync { if record.Jql != jql { record.Jql = jql - err = db.Update(&record, dal.Where("connection_id = ? AND board_id = ? ", data.Options.ConnectionId, data.Options.BoardId)) - if err != nil { - return errors.Default.Wrap(err, fmt.Sprintf("error updating record in _tool_jira_boards table for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) - } - logger.Info("full sync mode, update jql to %s", record.Jql) } + err = db.Update(&record, dal.Where("connection_id = ? AND board_id = ? ", data.Options.ConnectionId, data.Options.BoardId)) + if err != nil { + return errors.Default.Wrap(err, fmt.Sprintf("error updating record in _tool_jira_boards table for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) + } + logger.Info("full sync mode, update jql to %s", record.Jql) return nil } @@ -92,7 +100,6 @@ func CollectBoardFilterBegin(taskCtx plugin.SubTaskContext) errors.Error { flag := cfg.GetBool("JIRA_JQL_AUTO_FULL_REFRESH") if flag { logger.Info("connection_id:%d board_id:%d filter jql has changed, And the previous jql is %s, now jql is %s, run it in fullSync mode", data.Options.ConnectionId, data.Options.BoardId, record.Jql, jql) - // set full sync taskCtx.TaskContext().SetSyncPolicy(&coreModels.SyncPolicy{TriggerSyncPolicy: coreModels.TriggerSyncPolicy{FullSync: true}}) record.Jql = jql err = db.Update(&record, dal.Where("connection_id = ? AND board_id = ? ", data.Options.ConnectionId, data.Options.BoardId)) @@ -102,24 +109,28 @@ func CollectBoardFilterBegin(taskCtx plugin.SubTaskContext) errors.Error { } else { return errors.Default.New(fmt.Sprintf("connection_id:%d board_id:%d filter jql has changed, please use fullSync mode. And the previous jql is %s, now jql is %s", data.Options.ConnectionId, data.Options.BoardId, record.Jql, jql)) } + } else { + // JQL unchanged but sub-query may have changed — persist it + err = db.Update(&record, dal.Where("connection_id = ? AND board_id = ? ", data.Options.ConnectionId, data.Options.BoardId)) + if err != nil { + return errors.Default.Wrap(err, fmt.Sprintf("error updating record in _tool_jira_boards table for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) + } } - // no change return nil } -func getBoardFilterId(data *JiraTaskData) (string, error) { +func getBoardConfiguration(data *JiraTaskData) (*BoardConfiguration, error) { url := fmt.Sprintf("agile/1.0/board/%d/configuration", data.Options.BoardId) boardConfiguration, err := data.ApiClient.Get(url, nil, nil) if err != nil { - return "", err + return nil, err } bc := &BoardConfiguration{} err = helper.UnmarshalResponse(boardConfiguration, bc) if err != nil { - return "", err + return nil, err } - filterId := bc.Filter.ID - return filterId, nil + return bc, nil } func getBoardFilterJql(data *JiraTaskData, filterId string) (*FilterInfo, error) { @@ -141,6 +152,9 @@ type BoardConfiguration struct { Name string `json:"name"` Type string `json:"type"` Self string `json:"self"` + SubQuery struct { + Query string `json:"query"` + } `json:"subQuery"` Location struct { Type string `json:"type"` Key string `json:"key"` diff --git a/backend/plugins/jira/tasks/board_filter_begin_collector_test.go b/backend/plugins/jira/tasks/board_filter_begin_collector_test.go new file mode 100644 index 00000000000..dc81004a971 --- /dev/null +++ b/backend/plugins/jira/tasks/board_filter_begin_collector_test.go @@ -0,0 +1,137 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "testing" +) + +func Test_BoardConfiguration_UnmarshalSubQuery(t *testing.T) { + tests := []struct { + name string + raw string + wantSubQuery string + wantFilterID string + wantID int + wantName string + wantType string + wantColumnCount int + wantRankFieldID int + }{ + { + name: "kanban board with sub-filter object", + raw: `{"id":1201,"name":"Squad 5","type":"kanban",` + + `"self":"https://example.atlassian.net/rest/agile/1.0/board/1201/configuration",` + + `"filter":{"id":"17696","self":"https://example.atlassian.net/rest/api/2/filter/17696"},` + + `"subQuery":{"query":"fixVersion in unreleasedVersions() OR fixVersion is EMPTY"},` + + `"columnConfig":{"columns":[{"name":"Backlog","statuses":[{"id":"1","self":"https://example.atlassian.net/rest/api/2/status/1"}]},` + + `{"name":"Done","statuses":[{"id":"10037","self":"https://example.atlassian.net/rest/api/2/status/10037"}]}],` + + `"constraintType":"issueCount"},"ranking":{"rankCustomFieldId":10019}}`, + wantSubQuery: "fixVersion in unreleasedVersions() OR fixVersion is EMPTY", + wantFilterID: "17696", + wantID: 1201, + wantName: "Squad 5", + wantType: "kanban", + wantColumnCount: 2, + wantRankFieldID: 10019, + }, + { + name: "board without subQuery field", + raw: `{"id":500,"name":"No SubFilter Board","type":"scrum",` + + `"self":"https://example.atlassian.net/rest/agile/1.0/board/500/configuration",` + + `"filter":{"id":"99999","self":"https://example.atlassian.net/rest/api/2/filter/99999"},` + + `"columnConfig":{"columns":[],"constraintType":"issueCount"},"ranking":{"rankCustomFieldId":10019}}`, + wantSubQuery: "", + wantFilterID: "99999", + wantID: 500, + wantName: "No SubFilter Board", + wantType: "scrum", + wantColumnCount: 0, + wantRankFieldID: 10019, + }, + { + name: "board with empty subQuery object", + raw: `{"id":600,"name":"Empty SubQuery Board","type":"kanban",` + + `"self":"https://example.atlassian.net/rest/agile/1.0/board/600/configuration",` + + `"filter":{"id":"11111","self":"https://example.atlassian.net/rest/api/2/filter/11111"},` + + `"subQuery":{},` + + `"columnConfig":{"columns":[],"constraintType":"issueCount"},"ranking":{"rankCustomFieldId":10019}}`, + wantSubQuery: "", + wantFilterID: "11111", + wantID: 600, + wantName: "Empty SubQuery Board", + wantType: "kanban", + wantColumnCount: 0, + wantRankFieldID: 10019, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var bc BoardConfiguration + if err := json.Unmarshal([]byte(tt.raw), &bc); err != nil { + t.Fatalf("failed to unmarshal BoardConfiguration: %v", err) + } + if bc.SubQuery.Query != tt.wantSubQuery { + t.Errorf("SubQuery.Query = %q, want %q", bc.SubQuery.Query, tt.wantSubQuery) + } + if bc.Filter.ID != tt.wantFilterID { + t.Errorf("Filter.ID = %q, want %q", bc.Filter.ID, tt.wantFilterID) + } + if bc.ID != tt.wantID { + t.Errorf("ID = %d, want %d", bc.ID, tt.wantID) + } + if bc.Name != tt.wantName { + t.Errorf("Name = %q, want %q", bc.Name, tt.wantName) + } + if bc.Type != tt.wantType { + t.Errorf("Type = %q, want %q", bc.Type, tt.wantType) + } + if len(bc.ColumnConfig.Columns) != tt.wantColumnCount { + t.Errorf("ColumnConfig.Columns length = %d, want %d", len(bc.ColumnConfig.Columns), tt.wantColumnCount) + } + if bc.Ranking.RankCustomFieldID != tt.wantRankFieldID { + t.Errorf("Ranking.RankCustomFieldID = %d, want %d", bc.Ranking.RankCustomFieldID, tt.wantRankFieldID) + } + }) + } +} + +func Test_BoardConfiguration_FullJiraCloudResponse(t *testing.T) { + // Exact response payload from Jira Cloud for Board 1201 (Squad 5) + raw := `{"id":1201,"name":"Squad 5","type":"kanban","self":"https://rakutenadvertising.atlassian.net/rest/agile/1.0/board/1201/configuration","location":{"type":"user","id":"62d8159bb2e6b1992b5be875","self":"https://rakutenadvertising.atlassian.net/rest/api/2/user?accountId=62d8159bb2e6b1992b5be875"},"filter":{"id":"17696","self":"https://rakutenadvertising.atlassian.net/rest/api/2/filter/17696"},"subQuery":{"query":"fixVersion in unreleasedVersions() OR fixVersion is EMPTY"},"columnConfig":{"columns":[{"name":"Backlog","statuses":[{"id":"1","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/1"},{"id":"4","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/4"},{"id":"10016","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10016"},{"id":"10003","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10003"}]},{"name":"To Do","statuses":[{"id":"10054","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10054"}]},{"name":"Blocked","statuses":[{"id":"10019","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10019"}]},{"name":"In Development","statuses":[{"id":"10017","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10017"},{"id":"10177","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10177"},{"id":"10038","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10038"}]},{"name":"Code Review","statuses":[{"id":"10024","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10024"}]},{"name":"Ready for QA","statuses":[{"id":"10029","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10029"},{"id":"10033","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10033"}]},{"name":"In QA","statuses":[{"id":"10018","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10018"},{"id":"10158","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10158"}]},{"name":"Done","statuses":[{"id":"10037","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10037"}]}],"constraintType":"issueCount"},"ranking":{"rankCustomFieldId":10019}}` + + var bc BoardConfiguration + if err := json.Unmarshal([]byte(raw), &bc); err != nil { + t.Fatalf("failed to unmarshal real Jira Cloud response: %v", err) + } + + if bc.SubQuery.Query != "fixVersion in unreleasedVersions() OR fixVersion is EMPTY" { + t.Errorf("SubQuery.Query = %q, want the fixVersion sub-filter", bc.SubQuery.Query) + } + if bc.Filter.ID != "17696" { + t.Errorf("Filter.ID = %q, want %q", bc.Filter.ID, "17696") + } + if len(bc.ColumnConfig.Columns) != 8 { + t.Errorf("ColumnConfig.Columns length = %d, want 8", len(bc.ColumnConfig.Columns)) + } + if bc.Location.ID != "62d8159bb2e6b1992b5be875" { + t.Errorf("Location.ID = %q, want %q", bc.Location.ID, "62d8159bb2e6b1992b5be875") + } +} diff --git a/backend/plugins/jira/tasks/board_filter_end_collector.go b/backend/plugins/jira/tasks/board_filter_end_collector.go index 65d8eca14fd..4dfe06fefea 100644 --- a/backend/plugins/jira/tasks/board_filter_end_collector.go +++ b/backend/plugins/jira/tasks/board_filter_end_collector.go @@ -40,14 +40,13 @@ func CollectBoardFilterEnd(taskCtx plugin.SubTaskContext) errors.Error { db := taskCtx.GetDal() logger.Info("collect board in collectBoardFilterEnd: %d", data.Options.BoardId) - // get board filter id - filterId, err := getBoardFilterId(data) + boardConfig, err := getBoardConfiguration(data) if err != nil { - return errors.Default.Wrap(err, fmt.Sprintf("error getting board filter id for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) + return errors.Default.Wrap(err, fmt.Sprintf("error getting board configuration for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) } + filterId := boardConfig.Filter.ID logger.Info("collect board filter:%s", filterId) - // get board filter jql filterInfo, err := getBoardFilterJql(data, filterId) if err != nil { return errors.Default.Wrap(err, fmt.Sprintf("error getting board filter jql for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) @@ -55,7 +54,6 @@ func CollectBoardFilterEnd(taskCtx plugin.SubTaskContext) errors.Error { jql := filterInfo.Jql logger.Info("collect board filter jql:%s", jql) - // should not change var record models.JiraBoard err = db.First(&record, dal.Where("connection_id = ? AND board_id = ? ", data.Options.ConnectionId, data.Options.BoardId)) if err != nil { @@ -63,12 +61,21 @@ func CollectBoardFilterEnd(taskCtx plugin.SubTaskContext) errors.Error { } logger.Info("get board filter jql:%s", record.Jql) + cfg := taskCtx.GetConfigReader() + autoRefresh := cfg.GetBool("JIRA_JQL_AUTO_FULL_REFRESH") + if record.Jql != jql { - cfg := taskCtx.GetConfigReader() - flag := cfg.GetBool("JIRA_JQL_AUTO_FULL_REFRESH") - if !flag { + if !autoRefresh { return errors.Default.New(fmt.Sprintf("connection_id:%d board_id:%d filter jql has changed, please use fullSync mode. And the previous jql is %s, now jql is %s", data.Options.ConnectionId, data.Options.BoardId, record.Jql, jql)) } + logger.Warn(nil, "connection_id:%d board_id:%d filter jql changed during collection (previous: %s, now: %s)", data.Options.ConnectionId, data.Options.BoardId, record.Jql, jql) + } + + if record.SubQuery != boardConfig.SubQuery.Query { + logger.Warn(nil, "connection_id:%d board_id:%d board sub-filter changed during collection (previous: %s, now: %s)", data.Options.ConnectionId, data.Options.BoardId, record.SubQuery, boardConfig.SubQuery.Query) + if !autoRefresh { + return errors.Default.New(fmt.Sprintf("connection_id:%d board_id:%d board sub-filter has changed during collection, please use fullSync mode. Previous sub-filter: %s, now: %s", data.Options.ConnectionId, data.Options.BoardId, record.SubQuery, boardConfig.SubQuery.Query)) + } } return nil diff --git a/backend/plugins/jira/tasks/issue_collector.go b/backend/plugins/jira/tasks/issue_collector.go index 9a361cbbcca..997a7368598 100644 --- a/backend/plugins/jira/tasks/issue_collector.go +++ b/backend/plugins/jira/tasks/issue_collector.go @@ -18,20 +18,22 @@ limitations under the License. package tasks import ( + "bytes" "encoding/json" "fmt" "io" "net/http" "net/url" "strings" + "text/template" "time" "github.com/apache/incubator-devlake/core/dal" - "github.com/apache/incubator-devlake/plugins/jira/models" - "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/log" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/jira/models" ) const RAW_ISSUE_TABLE = "jira_api_issues" @@ -51,100 +53,215 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error { logger := taskCtx.GetLogger() apiCollector, err := api.NewStatefulApiCollector(api.RawDataSubTaskArgs{ Ctx: taskCtx, - /* - This struct will be JSONEncoded and stored into database along with raw data itself, to identity minimal - set of data to be process, for example, we process JiraIssues by Board - */ Params: JiraApiParams{ ConnectionId: data.Options.ConnectionId, BoardId: data.Options.BoardId, }, - /* - Table store raw data - */ Table: RAW_ISSUE_TABLE, }) if err != nil { return err } - // build jql - // IMPORTANT: we have to keep paginated data in a consistence order to avoid data-missing, if we sort issues by - // `updated`, issue will be jumping between pages if it got updated during the collection process + // IMPORTANT: we sort by `created ASC` to keep paginated data in a consistent order. + // Sorting by `updated` would cause issues to jump between pages during collection. loc, err := getTimeZone(taskCtx) if err != nil { logger.Info("failed to get timezone, err: %v", err) } else { logger.Info("got user's timezone: %v", loc.String()) } - jql := "ORDER BY created ASC" + incrementalJql := "ORDER BY created ASC" if apiCollector.GetSince() != nil { - jql = buildJQL(*apiCollector.GetSince(), loc) - } - - err = apiCollector.InitCollector(api.ApiCollectorArgs{ - ApiClient: data.ApiClient, - PageSize: data.Options.PageSize, - /* - url may use arbitrary variables from different connection in any order, we need GoTemplate to allow more - flexible for all kinds of possibility. - Pager contains information for a particular page, calculated by ApiCollector, and will be passed into - GoTemplate to generate a url for that page. - We want to do page-fetching in ApiCollector, because the logic are highly similar, by doing so, we can - avoid duplicate logic for every tasks, and when we have a better idea like improving performance, we can - do it in one place - */ - UrlTemplate: "agile/1.0/board/{{ .Params.BoardId }}/issue", - /* - (Optional) Return query string for request, or you can plug them into UrlTemplate directly - */ + incrementalJql = buildJQL(*apiCollector.GetSince(), loc) + } + + // Use the search API with `filter = {id}` JQL instead of the board Agile API. + // The board Agile API applies kanban sub-filters server-side, which silently + // excludes resolved issues (e.g. those with a released fixVersion). + // The search API with the saved filter JQL returns all matching issues. + var extraJql string + if data.Options.ScopeConfig != nil && data.Options.ScopeConfig.ExtraJQL != "" { + renderedJql, renderErr := renderExtraJQL(data.Options.ScopeConfig.ExtraJQL, data) + if renderErr != nil { + return renderErr + } + extraJql = renderedJql + } + filterJql := buildFilterJQL(data.FilterId, extraJql, incrementalJql) + logger.Info("collecting issues via search API with JQL: %s", filterJql) + + pageSize := data.Options.PageSize + if pageSize == 0 { + pageSize = 100 + } + + skipUnparseable := taskCtx.GetConfigReader().GetBool("JIRA_SKIP_UNPARSEABLE_ISSUES") + if skipUnparseable { + logger.Info("JIRA_SKIP_UNPARSEABLE_ISSUES is enabled, unparseable issue pages will be skipped") + } + + if strings.EqualFold(string(data.JiraServerInfo.DeploymentType), string(models.DeploymentServer)) { + logger.Info("Using api/2/search for JIRA Server issue collection") + err = setupIssueV2Collector(apiCollector, data, filterJql, pageSize, skipUnparseable, logger) + } else { + logger.Info("Using api/3/search/jql for JIRA Cloud issue collection") + err = setupIssueV3Collector(apiCollector, data, filterJql, pageSize, skipUnparseable, logger) + } + if err != nil { + return err + } + + return apiCollector.Execute() +} + +// JqlTemplateData holds the variables available inside an ExtraJQL template. +// Users reference these with Go template syntax, e.g. `{{.BoardName}}`. +type JqlTemplateData struct { + BoardId uint64 // numeric ID of the connected Jira board + BoardName string // display name of the connected Jira board +} + +// renderExtraJQL executes the ExtraJQL scope-config field as a Go text/template, +// substituting board-level variables so the same scope config can produce +// different JQL for different boards. +// +// The template is parsed with an empty FuncMap (no built-in helpers such as +// printf) and missingkey=error so that typos in variable names produce an +// explicit error rather than silently rendering "". +func renderExtraJQL(tmplStr string, data *JiraTaskData) (string, errors.Error) { + tmpl, err := template.New("extraJql"). + Funcs(template.FuncMap{}). + Option("missingkey=error"). + Parse(tmplStr) + if err != nil { + return "", errors.BadInput.Wrap(err, "invalid ExtraJQL template") + } + + vars := JqlTemplateData{ + BoardId: data.Options.BoardId, + } + if data.Board != nil { + vars.BoardName = data.Board.Name + } + + var buf bytes.Buffer + if execErr := tmpl.Execute(&buf, vars); execErr != nil { + return "", errors.BadInput.Wrap(execErr, "failed to render ExtraJQL template") + } + return buf.String(), nil +} + +// buildFilterJQL composes a final JQL query from three inputs: +// - filterId: a Jira saved-filter ID (referenced via `filter = {id}`) +// - extraJql: optional user-supplied JQL fragment appended as an AND condition +// (e.g. `project = "MyComponent"`) to scope a large board down to one project +// - incrementalJql: the time-based clause generated by buildJQL, always ending +// with "ORDER BY created ASC" +// +// extraJql is wrapped in parentheses so that any OR/NOT operators inside it +// do not interfere with the surrounding AND chain. +func buildFilterJQL(filterId string, extraJql string, incrementalJql string) string { + const orderBy = "ORDER BY created ASC" + + var conditions []string + if filterId != "" { + conditions = append(conditions, fmt.Sprintf("filter = %s", filterId)) + } + if extraJql != "" { + conditions = append(conditions, fmt.Sprintf("(%s)", extraJql)) + } + if incrementalJql != orderBy { + // strip the trailing " ORDER BY created ASC" to isolate the time condition + conditions = append(conditions, strings.TrimSuffix(incrementalJql, " "+orderBy)) + } + + if len(conditions) == 0 { + return orderBy + } + return strings.Join(conditions, " AND ") + " " + orderBy +} + +// parseIssuesResponse extracts the `issues` array from a Jira search response. +// +// A response that arrived with a successful status but carries a body which is not the +// expected JSON - a truncated payload, or an HTML error page substituted by a proxy - +// normally fails the whole collectIssues subtask, discarding an otherwise complete sync +// because of a single page. When skipUnparseable is set the page is logged and skipped +// instead. Non-2xx responses never reach this point; they are handled by the retry logic +// in the API client. +func parseIssuesResponse(res *http.Response, skipUnparseable bool, logger log.Logger) ([]json.RawMessage, errors.Error) { + blob, err := io.ReadAll(res.Body) + if err != nil { + return nil, errors.Convert(err) + } + var body struct { + Issues []json.RawMessage `json:"issues"` + } + if err := json.Unmarshal(blob, &body); err != nil { + if !skipUnparseable { + return nil, errors.Convert(err) + } + if logger != nil { + logger.Warn(err, "skipping unparseable issue page from %s (%d bytes)", responseUrl(res), len(blob)) + } + return []json.RawMessage{}, nil + } + return body.Issues, nil +} + +// responseUrl reports the request URL behind a response, for log messages. The request is +// always populated on responses returned by the API client, but a hand-built response in a +// test may not carry one. +func responseUrl(res *http.Response) string { + if res == nil || res.Request == nil || res.Request.URL == nil { + return "unknown url" + } + return res.Request.URL.String() +} + +func setupIssueV2Collector(apiCollector *api.StatefulApiCollector, data *JiraTaskData, filterJql string, pageSize int, skipUnparseable bool, logger log.Logger) errors.Error { + return apiCollector.InitCollector(api.ApiCollectorArgs{ + ApiClient: data.ApiClient, + PageSize: pageSize, + UrlTemplate: "api/2/search", Query: func(reqData *api.RequestData) (url.Values, errors.Error) { query := url.Values{} - query.Set("jql", jql) + query.Set("jql", filterJql) query.Set("startAt", fmt.Sprintf("%v", reqData.Pager.Skip)) query.Set("maxResults", fmt.Sprintf("%v", reqData.Pager.Size)) query.Set("expand", "changelog") return query, nil }, - /* - Some api might do pagination by http headers - */ - //Header: func(pager *plugin.Pager) http.Header { - //}, - /* - Sometimes, we need to collect data based on previous collected data, like jira changelog, it requires - issue_id as part of the url. - We can mimic `stdin` design, to accept a `Input` function which produces a `Iterator`, collector - should iterate all records, and do data-fetching for each on, either in parallel or sequential order - UrlTemplate: "api/3/issue/{{ Input.ID }}/changelog" - */ - //Input: databaseIssuesIterator, - /* - For api endpoint that returns number of total pages, ApiCollector can collect pages in parallel with ease, - or other techniques are required if this information was missing. - */ GetTotalPages: GetTotalPagesFromResponse, Concurrency: 10, ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { - var data struct { - Issues []json.RawMessage `json:"issues"` - } - blob, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Convert(err) - } - err = json.Unmarshal(blob, &data) - if err != nil { - return nil, errors.Convert(err) - } - return data.Issues, nil + return parseIssuesResponse(res, skipUnparseable, logger) }, }) - if err != nil { - return err - } +} - return apiCollector.Execute() +func setupIssueV3Collector(apiCollector *api.StatefulApiCollector, data *JiraTaskData, filterJql string, pageSize int, skipUnparseable bool, logger log.Logger) errors.Error { + return apiCollector.InitCollector(api.ApiCollectorArgs{ + ApiClient: data.ApiClient, + PageSize: pageSize, + UrlTemplate: "api/3/search/jql", + GetNextPageCustomData: getNextPageCustomDataForV3, + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("jql", filterJql) + query.Set("maxResults", fmt.Sprintf("%v", reqData.Pager.Size)) + query.Set("expand", "changelog") + query.Set("fields", "*all") + if reqData.CustomData != nil { + query.Set("nextPageToken", reqData.CustomData.(string)) + } + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + return parseIssuesResponse(res, skipUnparseable, logger) + }, + }) } // buildJQL build jql based on timeAfter and incremental mode diff --git a/backend/plugins/jira/tasks/issue_collector_test.go b/backend/plugins/jira/tasks/issue_collector_test.go index 99bf5a53367..098196f9354 100644 --- a/backend/plugins/jira/tasks/issue_collector_test.go +++ b/backend/plugins/jira/tasks/issue_collector_test.go @@ -18,8 +18,17 @@ limitations under the License. package tasks import ( + "io" + "net/http" + "net/url" + "strings" "testing" "time" + + "github.com/apache/incubator-devlake/helpers/unithelper" + mocklog "github.com/apache/incubator-devlake/mocks/core/log" + "github.com/apache/incubator-devlake/plugins/jira/models" + "github.com/stretchr/testify/mock" ) func Test_buildJQL(t *testing.T) { @@ -61,3 +70,259 @@ func Test_buildJQL(t *testing.T) { }) } } + +func Test_buildFilterJQL(t *testing.T) { + tests := []struct { + name string + filterId string + extraJql string + incrementalJql string + want string + }{ + { + name: "full sync with filter", + filterId: "12345", + incrementalJql: "ORDER BY created ASC", + want: "filter = 12345 ORDER BY created ASC", + }, + { + name: "incremental sync with filter", + filterId: "12345", + incrementalJql: "updated >= '2021/02/05 12:05' ORDER BY created ASC", + want: "filter = 12345 AND updated >= '2021/02/05 12:05' ORDER BY created ASC", + }, + { + name: "empty filter id falls back to incremental only", + filterId: "", + incrementalJql: "ORDER BY created ASC", + want: "ORDER BY created ASC", + }, + { + name: "empty filter id with incremental clause", + filterId: "", + incrementalJql: "updated >= '2024/01/01 00:00' ORDER BY created ASC", + want: "updated >= '2024/01/01 00:00' ORDER BY created ASC", + }, + { + name: "extra jql with filter full sync", + filterId: "12345", + extraJql: `project = "MyComponent"`, + incrementalJql: "ORDER BY created ASC", + want: `filter = 12345 AND (project = "MyComponent") ORDER BY created ASC`, + }, + { + name: "extra jql with filter incremental sync", + filterId: "12345", + extraJql: `project = "MyComponent"`, + incrementalJql: "updated >= '2024/01/01 00:00' ORDER BY created ASC", + want: `filter = 12345 AND (project = "MyComponent") AND updated >= '2024/01/01 00:00' ORDER BY created ASC`, + }, + { + name: "extra jql without filter", + filterId: "", + extraJql: `project = "MyComponent"`, + incrementalJql: "ORDER BY created ASC", + want: `(project = "MyComponent") ORDER BY created ASC`, + }, + { + name: "extra jql without filter, incremental sync", + filterId: "", + extraJql: `project = "MyComponent"`, + incrementalJql: "updated >= '2024/01/01 00:00' ORDER BY created ASC", + want: `(project = "MyComponent") AND updated >= '2024/01/01 00:00' ORDER BY created ASC`, + }, + { + name: "extra jql with OR operator is parenthesized", + filterId: "12345", + extraJql: `project = "A" OR project = "B"`, + incrementalJql: "ORDER BY created ASC", + want: `filter = 12345 AND (project = "A" OR project = "B") ORDER BY created ASC`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := buildFilterJQL(tt.filterId, tt.extraJql, tt.incrementalJql); got != tt.want { + t.Errorf("buildFilterJQL() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_parseIssuesResponse(t *testing.T) { + const twoIssues = `{"issues":[{"id":"1"},{"id":"2"}]}` + // A proxy returning an HTML error page under a 200, the case reported in #8949. + const htmlPage = `502 Bad Gateway` + // A response truncated mid-flight, so the JSON never terminates. + const truncated = `{"issues":[{"id":"1"},{"id":` + + makeResponse := func(body string) *http.Response { + return &http.Response{ + Body: io.NopCloser(strings.NewReader(body)), + Request: &http.Request{URL: &url.URL{Scheme: "https", Host: "jira.example.com", Path: "/rest/api/2/search"}}, + } + } + + tests := []struct { + name string + body string + skipUnparseable bool + wantCount int + wantErr bool + // wantSkipped marks the cases that took the skip path, which must yield an + // empty non-nil slice so the collector records "this page had no rows" rather + // than treating the result as absent. + wantSkipped bool + }{ + { + name: "valid page is parsed", + body: twoIssues, + wantCount: 2, + }, + { + name: "valid page is parsed when skipping is enabled", + body: twoIssues, + skipUnparseable: true, + wantCount: 2, + }, + { + name: "html body fails by default", + body: htmlPage, + wantErr: true, + }, + { + name: "html body is skipped when enabled", + body: htmlPage, + skipUnparseable: true, + wantCount: 0, + wantSkipped: true, + }, + { + name: "truncated body fails by default", + body: truncated, + wantErr: true, + }, + { + name: "truncated body is skipped when enabled", + body: truncated, + skipUnparseable: true, + wantCount: 0, + wantSkipped: true, + }, + { + name: "valid page without an issues key yields no issues", + body: `{"total":0}`, + wantCount: 0, + }, + } + + // unithelper.DummyLogger stubs Warn with two arguments, but the real signature is + // Warn(err, format, a ...interface{}), which mockery records as three. Add the + // variadic form so the skip path can log. + newLogger := func() *mocklog.Logger { + logger := unithelper.DummyLogger() + logger.On("Warn", mock.Anything, mock.Anything, mock.Anything).Maybe() + return logger + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseIssuesResponse(makeResponse(tt.body), tt.skipUnparseable, newLogger()) + if (err != nil) != tt.wantErr { + t.Errorf("parseIssuesResponse() error = %v, wantErr %v", err, tt.wantErr) + return + } + if tt.wantErr { + return + } + if len(got) != tt.wantCount { + t.Errorf("parseIssuesResponse() returned %d issues, want %d", len(got), tt.wantCount) + } + if tt.wantSkipped && got == nil { + t.Error("parseIssuesResponse() returned nil for a skipped page, want an empty slice") + } + }) + } +} + +func Test_responseUrl(t *testing.T) { + withUrl := &http.Response{ + Request: &http.Request{URL: &url.URL{Scheme: "https", Host: "jira.example.com", Path: "/rest/api/2/search"}}, + } + if got, want := responseUrl(withUrl), "https://jira.example.com/rest/api/2/search"; got != want { + t.Errorf("responseUrl() = %v, want %v", got, want) + } + if got, want := responseUrl(&http.Response{}), "unknown url"; got != want { + t.Errorf("responseUrl() with no request = %v, want %v", got, want) + } + if got, want := responseUrl(nil), "unknown url"; got != want { + t.Errorf("responseUrl(nil) = %v, want %v", got, want) + } +} + +func Test_renderExtraJQL(t *testing.T) { + makeData := func(boardId uint64, boardName string, _ string) *JiraTaskData { + return &JiraTaskData{ + Options: &JiraOptions{BoardId: boardId}, + Board: &models.JiraBoard{BoardId: boardId, Name: boardName}, + } + } + + tests := []struct { + name string + tmpl string + data *JiraTaskData + want string + wantErr bool + }{ + { + name: "static JQL passes through unchanged", + tmpl: `project = "MyProject"`, + data: makeData(1, "My Board", ""), + want: `project = "MyProject"`, + }, + { + name: "BoardName substitution", + tmpl: `project = "{{.BoardName}}"`, + data: makeData(42, "Team Alpha", ""), + want: `project = "Team Alpha"`, + }, + { + name: "BoardId substitution", + tmpl: `cf[10001] = {{.BoardId}}`, + data: makeData(99, "Some Board", ""), + want: `cf[10001] = 99`, + }, + { + name: "nil Board falls back to empty BoardName", + tmpl: `project = "{{.BoardName}}"`, + data: &JiraTaskData{Options: &JiraOptions{BoardId: 1}, Board: nil}, + want: `project = ""`, + }, + { + name: "invalid template returns error", + tmpl: `project = "{{.Unclosed"`, + data: makeData(1, "My Board", ""), + wantErr: true, + }, + { + name: "unknown field returns error (missingkey=error)", + tmpl: `component = "{{.Typo}}"`, + data: makeData(1, "My Board", ""), + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := renderExtraJQL(tt.tmpl, tt.data) + if (err != nil) != tt.wantErr { + t.Errorf("renderExtraJQL() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !tt.wantErr && got != tt.want { + t.Errorf("renderExtraJQL() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/backend/plugins/jira/tasks/sprint_report_collector.go b/backend/plugins/jira/tasks/sprint_report_collector.go new file mode 100644 index 00000000000..1f8db1f5cf9 --- /dev/null +++ b/backend/plugins/jira/tasks/sprint_report_collector.go @@ -0,0 +1,116 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/jira/tasks/apiv2models" +) + +const RAW_SPRINT_REPORT_TABLE = "jira_api_sprint_reports" + +var _ plugin.SubTaskEntryPoint = CollectSprintReport + +var CollectSprintReportMeta = plugin.SubTaskMeta{ + Name: "collectSprintReport", + EntryPoint: CollectSprintReport, + EnabledByDefault: true, + Description: "collect Jira Sprint Report, the frozen committed/completed snapshot Jira takes at sprint close", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +// CollectSprintReport fetches the Greenhopper Sprint Report for every closed +// sprint on this board. It depends on CollectSprints/ExtractSprints having +// already populated _tool_jira_board_sprints and _tool_jira_sprints, since +// that's where the (boardId, sprintId) pairs to query come from. Only closed +// sprints are queried: the report is only a meaningful, frozen snapshot once +// a sprint has actually closed. +func CollectSprintReport(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*JiraTaskData) + db := taskCtx.GetDal() + logger := taskCtx.GetLogger() + + apiCollector, err := api.NewStatefulApiCollector(api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: JiraApiParams{ + ConnectionId: data.Options.ConnectionId, + BoardId: data.Options.BoardId, + }, + Table: RAW_SPRINT_REPORT_TABLE, + }) + if err != nil { + return err + } + + clauses := []dal.Clause{ + dal.Select("bs.board_id AS board_id, bs.sprint_id AS sprint_id, s.complete_date AS update_time"), + dal.From("_tool_jira_board_sprints bs"), + dal.Join("LEFT JOIN _tool_jira_sprints s ON (bs.connection_id = s.connection_id AND bs.sprint_id = s.sprint_id)"), + dal.Where("bs.connection_id = ? AND bs.board_id = ? AND s.state = ?", data.Options.ConnectionId, data.Options.BoardId, "closed"), + } + if apiCollector.IsIncremental() && apiCollector.GetSince() != nil { + clauses = append(clauses, dal.Where("s.complete_date > ?", apiCollector.GetSince())) + } + cursor, err := db.Cursor(clauses...) + if err != nil { + logger.Error(err, "collect sprint report error") + return err + } + + iterator, err := api.NewDalCursorIterator(db, cursor, reflect.TypeOf(apiv2models.SprintReportInput{})) + if err != nil { + return err + } + + err = apiCollector.InitCollector(api.ApiCollectorArgs{ + ApiClient: data.ApiClient, + Input: iterator, + // e.g. https://xxx.atlassian.net/rest/greenhopper/1.0/rapid/charts/sprintreport?rapidViewId=1&sprintId=2 + UrlTemplate: "greenhopper/1.0/rapid/charts/sprintreport", + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + input := reqData.Input.(*apiv2models.SprintReportInput) + query := url.Values{} + query.Set("rapidViewId", fmt.Sprintf("%d", input.BoardId)) + query.Set("sprintId", fmt.Sprintf("%d", input.SprintId)) + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + blob, e := io.ReadAll(res.Body) + if e != nil { + return nil, errors.Convert(e) + } + return []json.RawMessage{blob}, nil + }, + AfterResponse: ignoreHTTPStatus400, + }) + if err != nil { + return err + } + + return apiCollector.Execute() +} diff --git a/backend/plugins/jira/tasks/sprint_report_convertor.go b/backend/plugins/jira/tasks/sprint_report_convertor.go new file mode 100644 index 00000000000..76046c27319 --- /dev/null +++ b/backend/plugins/jira/tasks/sprint_report_convertor.go @@ -0,0 +1,99 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/jira/models" +) + +var ConvertSprintReportMeta = plugin.SubTaskMeta{ + Name: "convertSprintReport", + EntryPoint: ConvertSprintReport, + EnabledByDefault: true, + Description: "aggregate Jira Sprint Report into committed/completed story points on the domain sprints table", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +type sprintVelocity struct { + SprintId uint64 `json:"sprint_id"` + Committed *float64 `json:"committed"` + Completed *float64 `json:"completed"` +} + +// ConvertSprintReport aggregates _tool_jira_sprint_reports (one row per +// issue per sprint per bucket) into two numbers per sprint: +// - committed: sum of each issue's start-of-sprint estimate, for every +// issue that was part of the sprint when it began (completed, +// notCompleted, and punted issues; punted issues were still committed, +// they were just removed before the sprint closed). +// - completed: sum of each issue's end-of-sprint estimate, for issues in +// the completed bucket only. +// +// It writes these directly onto the existing domain sprints row via +// UpdateColumns rather than going through the usual convert-and-save +// pipeline, specifically so it only touches these two columns and can't +// clobber the Name/Url/Status/dates that ConvertSprints already wrote. +func ConvertSprintReport(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*JiraTaskData) + db := taskCtx.GetDal() + connectionId := data.Options.ConnectionId + sprintIdGen := didgen.NewDomainIdGenerator(&models.JiraSprint{}) + + clauses := []dal.Clause{ + dal.Select(` + sprint_id, + SUM(CASE WHEN bucket IN (?, ?, ?) THEN story_points_at_sprint_start ELSE NULL END) AS committed, + SUM(CASE WHEN bucket = ? THEN story_points_at_sprint_end ELSE NULL END) AS completed + `, + models.SprintReportBucketCompleted, + models.SprintReportBucketNotCompleted, + models.SprintReportBucketPunted, + models.SprintReportBucketCompleted, + ), + dal.From("_tool_jira_sprint_reports"), + dal.Where("connection_id = ? AND board_id = ?", connectionId, data.Options.BoardId), + dal.Groupby("sprint_id"), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + defer cursor.Close() + + for cursor.Next() { + var row sprintVelocity + if err = db.Fetch(cursor, &row); err != nil { + return err + } + domainSprintId := sprintIdGen.Generate(connectionId, row.SprintId) + updateSet := []dal.DalSet{ + {ColumnName: "committed_story_point", Value: row.Committed}, + {ColumnName: "completed_story_point", Value: row.Completed}, + } + if err = db.UpdateColumns(&ticket.Sprint{}, updateSet, dal.Where("id = ?", domainSprintId)); err != nil { + return err + } + } + + return nil +} diff --git a/backend/plugins/jira/tasks/sprint_report_extractor.go b/backend/plugins/jira/tasks/sprint_report_extractor.go new file mode 100644 index 00000000000..c24c6be36d9 --- /dev/null +++ b/backend/plugins/jira/tasks/sprint_report_extractor.go @@ -0,0 +1,92 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/jira/models" + "github.com/apache/incubator-devlake/plugins/jira/tasks/apiv2models" +) + +var _ plugin.SubTaskEntryPoint = ExtractSprintReport + +var ExtractSprintReportMeta = plugin.SubTaskMeta{ + Name: "extractSprintReport", + EntryPoint: ExtractSprintReport, + EnabledByDefault: true, + Description: "extract Jira Sprint Report", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ExtractSprintReport(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*JiraTaskData) + connectionId := data.Options.ConnectionId + + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: JiraApiParams{ + ConnectionId: data.Options.ConnectionId, + BoardId: data.Options.BoardId, + }, + Table: RAW_SPRINT_REPORT_TABLE, + }, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + var report apiv2models.SprintReport + if err := errors.Convert(json.Unmarshal(row.Data, &report)); err != nil { + return nil, err + } + var input apiv2models.SprintReportInput + if err := errors.Convert(json.Unmarshal(row.Input, &input)); err != nil { + return nil, err + } + + var result []interface{} + addBucket := func(issues []apiv2models.SprintReportIssue, bucket string) { + for _, issue := range issues { + result = append(result, &models.JiraSprintReport{ + ConnectionId: connectionId, + BoardId: input.BoardId, + SprintId: input.SprintId, + IssueId: issue.Id, + IssueKey: issue.Key, + Bucket: bucket, + Done: issue.Done, + StoryPointsAtSprintStart: issue.EstimateStatistic.StatFieldValue.Value, + StoryPointsAtSprintEnd: issue.CurrentEstimateStatistic.StatFieldValue.Value, + }) + } + } + addBucket(report.Contents.CompletedIssues, models.SprintReportBucketCompleted) + addBucket(report.Contents.IssuesNotCompletedInCurrentSprint, models.SprintReportBucketNotCompleted) + addBucket(report.Contents.PuntedIssues, models.SprintReportBucketPunted) + addBucket(report.Contents.IssuesCompletedInAnotherSprint, models.SprintReportBucketCompletedInOtherSprint) + + return result, nil + }, + }) + if err != nil { + return err + } + + return extractor.Execute() +} diff --git a/backend/plugins/jira/tasks/task_data.go b/backend/plugins/jira/tasks/task_data.go index 1b0580396c5..3505bf162e8 100644 --- a/backend/plugins/jira/tasks/task_data.go +++ b/backend/plugins/jira/tasks/task_data.go @@ -37,6 +37,8 @@ type JiraTaskData struct { Options *JiraOptions ApiClient *api.ApiAsyncClient JiraServerInfo models.JiraServerInfo + FilterId string + Board *models.JiraBoard } type JiraApiParams models.JiraApiParams diff --git a/backend/plugins/linear/e2e/snapshot_tables/sprints.csv b/backend/plugins/linear/e2e/snapshot_tables/sprints.csv index a09ab2ffe70..c6324bf0200 100644 --- a/backend/plugins/linear/e2e/snapshot_tables/sprints.csv +++ b/backend/plugins/linear/e2e/snapshot_tables/sprints.csv @@ -1,3 +1,3 @@ -id,name,url,status,started_date,ended_date,completed_date,original_board_id -linear:LinearCycle:1:cycle-1,Cycle 1,,CLOSED,2026-04-20T00:00:00.000+00:00,2026-05-04T00:00:00.000+00:00,2026-05-04T00:00:00.000+00:00,linear:LinearTeam:1:team-1 -linear:LinearCycle:1:cycle-2,Sprint 2,,ACTIVE,2026-05-04T00:00:00.000+00:00,2026-05-18T00:00:00.000+00:00,,linear:LinearTeam:1:team-1 +id,name,url,status,started_date,ended_date,completed_date,original_board_id,committed_story_point,completed_story_point +linear:LinearCycle:1:cycle-1,Cycle 1,,CLOSED,2026-04-20T00:00:00.000+00:00,2026-05-04T00:00:00.000+00:00,2026-05-04T00:00:00.000+00:00,linear:LinearTeam:1:team-1,, +linear:LinearCycle:1:cycle-2,Sprint 2,,ACTIVE,2026-05-04T00:00:00.000+00:00,2026-05-18T00:00:00.000+00:00,,linear:LinearTeam:1:team-1,, diff --git a/backend/plugins/opsgenie/api/connection_api.go b/backend/plugins/opsgenie/api/connection_api.go index c2844efaa1c..a56118cc73d 100644 --- a/backend/plugins/opsgenie/api/connection_api.go +++ b/backend/plugins/opsgenie/api/connection_api.go @@ -48,7 +48,7 @@ func testOpsgenieConn(ctx context.Context, connection models.OpsgenieConn) (*plu } if response.StatusCode == http.StatusForbidden { - return nil, errors.HttpStatus(http.StatusForbidden).New("API Key need 'Read' and 'Configuration access' Access rights") + return nil, errors.HttpStatus(http.StatusForbidden).New("API Key needs 'Read', 'Create/Update', and 'Configuration Access' access rights") } if response.StatusCode == http.StatusOK || response.StatusCode == http.StatusAccepted { diff --git a/backend/plugins/org/impl/impl.go b/backend/plugins/org/impl/impl.go index e68257eec31..39d020fac5d 100644 --- a/backend/plugins/org/impl/impl.go +++ b/backend/plugins/org/impl/impl.go @@ -102,24 +102,41 @@ func (p Org) RootPkgPath() string { return "github.com/apache/incubator-devlake/plugins/org" } -func (p Org) ApiResources() map[string]map[string]plugin.ApiResourceHandler { +// wrapHandler defers the resolution of p.handlers to request time. +// ApiResources() may be evaluated during route registration, which can happen +// before InitPlugins() has called Init() (InitPlugins runs inside +// pipelineServiceInit); a bound method value like p.handlers.GetTeam would +// capture a nil receiver permanently, making every org endpoint panic with a +// nil pointer dereference. See https://github.com/apache/devlake/issues/9021. +func (p *Org) wrapHandler( + method func(*api.Handlers, *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error), +) plugin.ApiResourceHandler { + return func(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + if p.handlers == nil { + return nil, errors.Internal.New("org plugin is not initialized yet, please retry later") + } + return method(p.handlers, input) + } +} + +func (p *Org) ApiResources() map[string]map[string]plugin.ApiResourceHandler { return map[string]map[string]plugin.ApiResourceHandler{ "teams.csv": { - "GET": p.handlers.GetTeam, - "PUT": p.handlers.CreateTeam, + "GET": p.wrapHandler((*api.Handlers).GetTeam), + "PUT": p.wrapHandler((*api.Handlers).CreateTeam), }, "users.csv": { - "GET": p.handlers.GetUser, - "PUT": p.handlers.CreateUser, + "GET": p.wrapHandler((*api.Handlers).GetUser), + "PUT": p.wrapHandler((*api.Handlers).CreateUser), }, "user_account_mapping.csv": { - "GET": p.handlers.GetUserAccountMapping, - "PUT": p.handlers.CreateUserAccountMapping, + "GET": p.wrapHandler((*api.Handlers).GetUserAccountMapping), + "PUT": p.wrapHandler((*api.Handlers).CreateUserAccountMapping), }, "project_mapping.csv": { - "GET": p.handlers.GetProjectMapping, - "PUT": p.handlers.CreateProjectMapping, + "GET": p.wrapHandler((*api.Handlers).GetProjectMapping), + "PUT": p.wrapHandler((*api.Handlers).CreateProjectMapping), }, } } diff --git a/backend/plugins/org/impl/impl_test.go b/backend/plugins/org/impl/impl_test.go new file mode 100644 index 00000000000..4b6313ee33e --- /dev/null +++ b/backend/plugins/org/impl/impl_test.go @@ -0,0 +1,42 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/plugin" + "github.com/stretchr/testify/assert" +) + +// Route registration may evaluate ApiResources() before InitPlugins() has run +// (see https://github.com/apache/devlake/issues/9021). Handlers obtained from +// an uninitialized plugin must fail gracefully instead of panicking with a +// nil pointer dereference, and keep working once Init() runs later. +func TestApiResourcesBeforeInitFailsGracefully(t *testing.T) { + p := &Org{} + for resource, methods := range p.ApiResources() { + for method, handler := range methods { + assert.NotPanics(t, func() { + out, err := handler(&plugin.ApiResourceInput{}) + assert.Nilf(t, out, "%s %s should return no output before Init", method, resource) + assert.NotNilf(t, err, "%s %s should return an error before Init", method, resource) + }, "%s %s must not panic before Init", method, resource) + } + } +} diff --git a/backend/plugins/schema_e2e/migration_schema_test.go b/backend/plugins/schema_e2e/migration_schema_test.go new file mode 100644 index 00000000000..0fbc02893bc --- /dev/null +++ b/backend/plugins/schema_e2e/migration_schema_test.go @@ -0,0 +1,242 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package schema_e2e contains a cross-plugin regression guard that runs the +// REAL migration scripts of every built-in Go plugin and then asserts that the +// resulting database schema still matches what each runtime GORM model expects. +// +// It lives in an `e2e` package on purpose: it needs a real database +// (E2E_DB_URL) and is therefore only executed by `make e2e-test-go-plugins` +// (scripts/e2e-test-go-plugins.sh selects packages whose import path contains +// "e2e"), and excluded from the DB-less unit test run +// (scripts/unit-test-go.sh skips paths matching "e2e"). +package schema_e2e + +import ( + "os" + "path/filepath" + "sync" + "testing" + + "github.com/apache/incubator-devlake/core/config" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/migration" + coreMigration "github.com/apache/incubator-devlake/core/models/migrationscripts" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/impls/dalgorm" + "github.com/apache/incubator-devlake/impls/logruslog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm/schema" + + ae "github.com/apache/incubator-devlake/plugins/ae/impl" + argocd "github.com/apache/incubator-devlake/plugins/argocd/impl" + asana "github.com/apache/incubator-devlake/plugins/asana/impl" + azuredevops "github.com/apache/incubator-devlake/plugins/azuredevops_go/impl" + bamboo "github.com/apache/incubator-devlake/plugins/bamboo/impl" + bitbucket "github.com/apache/incubator-devlake/plugins/bitbucket/impl" + bitbucket_server "github.com/apache/incubator-devlake/plugins/bitbucket_server/impl" + circleci "github.com/apache/incubator-devlake/plugins/circleci/impl" + claudeCode "github.com/apache/incubator-devlake/plugins/claude_code/impl" + clickup "github.com/apache/incubator-devlake/plugins/clickup/impl" + customize "github.com/apache/incubator-devlake/plugins/customize/impl" + dbt "github.com/apache/incubator-devlake/plugins/dbt/impl" + dora "github.com/apache/incubator-devlake/plugins/dora/impl" + feishu "github.com/apache/incubator-devlake/plugins/feishu/impl" + copilot "github.com/apache/incubator-devlake/plugins/gh-copilot/impl" + gitee "github.com/apache/incubator-devlake/plugins/gitee/impl" + gitextractor "github.com/apache/incubator-devlake/plugins/gitextractor/impl" + github "github.com/apache/incubator-devlake/plugins/github/impl" + githubGraphql "github.com/apache/incubator-devlake/plugins/github_graphql/impl" + gitlab "github.com/apache/incubator-devlake/plugins/gitlab/impl" + icla "github.com/apache/incubator-devlake/plugins/icla/impl" + incidentio "github.com/apache/incubator-devlake/plugins/incidentio/impl" + issueTrace "github.com/apache/incubator-devlake/plugins/issue_trace/impl" + jenkins "github.com/apache/incubator-devlake/plugins/jenkins/impl" + jira "github.com/apache/incubator-devlake/plugins/jira/impl" + linear "github.com/apache/incubator-devlake/plugins/linear/impl" + linker "github.com/apache/incubator-devlake/plugins/linker/impl" + opsgenie "github.com/apache/incubator-devlake/plugins/opsgenie/impl" + org "github.com/apache/incubator-devlake/plugins/org/impl" + pagerduty "github.com/apache/incubator-devlake/plugins/pagerduty/impl" + q_dev "github.com/apache/incubator-devlake/plugins/q_dev/impl" + refdiff "github.com/apache/incubator-devlake/plugins/refdiff/impl" + rootly "github.com/apache/incubator-devlake/plugins/rootly/impl" + slack "github.com/apache/incubator-devlake/plugins/slack/impl" + sonarqube "github.com/apache/incubator-devlake/plugins/sonarqube/impl" + starrocks "github.com/apache/incubator-devlake/plugins/starrocks/impl" + taiga "github.com/apache/incubator-devlake/plugins/taiga/impl" + tapd "github.com/apache/incubator-devlake/plugins/tapd/impl" + teambition "github.com/apache/incubator-devlake/plugins/teambition/impl" + tempo "github.com/apache/incubator-devlake/plugins/tempo/impl" + testmo "github.com/apache/incubator-devlake/plugins/testmo/impl" + trello "github.com/apache/incubator-devlake/plugins/trello/impl" + webhook "github.com/apache/incubator-devlake/plugins/webhook/impl" + zentao "github.com/apache/incubator-devlake/plugins/zentao/impl" +) + +// allGoPlugins lists EVERY built-in Go plugin. Keep it in sync with the plugin +// directories under backend/plugins/ (the TestAllGoPluginsListed guard below +// fails if a new plugin's `impl` package is added but not registered here). +func allGoPlugins() []plugin.PluginMeta { + return []plugin.PluginMeta{ + ae.AE{}, + argocd.ArgoCD{}, + asana.Asana{}, + azuredevops.Azuredevops{}, + bamboo.Bamboo{}, + bitbucket.Bitbucket{}, + bitbucket_server.BitbucketServer{}, + circleci.Circleci{}, + claudeCode.ClaudeCode{}, + clickup.ClickUp{}, + customize.Customize{}, + dbt.Dbt{}, + dora.Dora{}, + feishu.Feishu{}, + copilot.GhCopilot{}, + gitee.Gitee{}, + gitextractor.GitExtractor{}, + github.Github{}, + githubGraphql.GithubGraphql{}, + gitlab.Gitlab{}, + icla.Icla{}, + incidentio.Incidentio{}, + issueTrace.IssueTrace{}, + jenkins.Jenkins{}, + jira.Jira{}, + linear.Linear{}, + linker.Linker{}, + opsgenie.Opsgenie{}, + org.Org{}, + pagerduty.PagerDuty{}, + q_dev.QDev{}, + refdiff.RefDiff{}, + rootly.Rootly{}, + slack.Slack{}, + sonarqube.Sonarqube{}, + starrocks.StarRocks{}, + taiga.Taiga{}, + tapd.Tapd{}, + teambition.Teambition{}, + tempo.Tempo{}, + testmo.Testmo{}, + trello.Trello{}, + webhook.Webhook{}, + zentao.Zentao{}, + } +} + +// TestAllGoPluginsListed guarantees allGoPlugins() stays complete: it counts the +// plugin directories that ship an `impl` package and fails if that number does +// not match the registered list. This makes the schema-drift guard below +// automatically cover any newly added plugin. +func TestAllGoPluginsListed(t *testing.T) { + entries, err := os.ReadDir("..") + require.NoError(t, err) + dirsWithImpl := 0 + for _, e := range entries { + if !e.IsDir() { + continue + } + if info, statErr := os.Stat(filepath.Join("..", e.Name(), "impl")); statErr == nil && info.IsDir() { + dirsWithImpl++ + } + } + assert.Equalf(t, dirsWithImpl, len(allGoPlugins()), + "number of plugin dirs with an impl/ package (%d) != registered plugins (%d); "+ + "add the new plugin to allGoPlugins() in plugins/schema_e2e/migration_schema_test.go", + dirsWithImpl, len(allGoPlugins())) +} + +// TestMigrationSchemaMatchesModels applies the real framework + plugin migration +// scripts and then verifies, for every plugin model, that each column the +// runtime GORM model declares actually exists in the migrated table. +// +// This is a cross-plugin generalization of the Jira Sprint Report regression: +// a migration created `_tool_jira_sprint_reports` without embedding +// common.NoPKModel, so the `_raw_data_*` columns were missing and the +// ApiExtractor cleanup query failed at runtime with +// "Unknown column '_raw_data_table' in 'where clause'". +// +// Tables that no migration creates (e.g. models materialized lazily at runtime) +// are skipped, so the check specifically targets *drift* between an existing +// table and its model — which is exactly the failure mode above. +// +// The migrations run against a dedicated, empty database (see +// e2ehelper.NewIsolatedMigrationDb) because the shared e2e database is polluted +// by the other e2e tests, which AutoMigrate tables without recording anything +// in `_devlake_migration_history`. +func TestMigrationSchemaMatchesModels(t *testing.T) { + db := e2ehelper.NewIsolatedMigrationDb(t, "schema_drift") + dalInstance := dalgorm.NewDalgorm(db) + basicRes := runner.CreateBasicRes(config.GetConfig(), logruslog.Global, db) + + // Apply the migrations exactly the way the server does on startup. + migrator, migErr := migration.NewMigrator(basicRes) + require.NoError(t, migErr) + migrator.Register(coreMigration.All(), "Framework") + for _, p := range allGoPlugins() { + if migratable, ok := p.(plugin.PluginMigration); ok { + migrator.Register(migratable.MigrationScripts(), p.Name()) + } + } + require.NoError(t, migrator.Execute()) + + keepAll := func(dal.ColumnMeta) bool { return true } + + for _, p := range allGoPlugins() { + modeler, ok := p.(plugin.PluginModel) + if !ok { + continue + } + p := p + t.Run(p.Name(), func(t *testing.T) { + for _, table := range modeler.GetTablesInfo() { + table := table + // Columns that actually exist in the migrated table. + actualColumns, colErr := dal.GetColumnNames(dalInstance, table, keepAll) + if colErr != nil || len(actualColumns) == 0 { + // No migration created this table (e.g. runtime-only / + // dynamic model) — nothing to validate for drift. + t.Logf("skip %q: table not present after migrations", table.TableName()) + continue + } + existing := make(map[string]struct{}, len(actualColumns)) + for _, c := range actualColumns { + existing[c] = struct{}{} + } + + // Columns the runtime GORM model expects. + sch, parseErr := schema.Parse(table, &sync.Map{}, schema.NamingStrategy{}) + require.NoErrorf(t, parseErr, "unable to parse schema for %T", table) + for _, field := range sch.Fields { + if field.DBName == "" || field.IgnoreMigration { + continue + } + _, present := existing[field.DBName] + assert.Truef(t, present, + "[%s] table %q is missing column %q expected by model %T — "+ + "did a migration script forget to embed common.NoPKModel (raw-data columns) or add the field?", + p.Name(), table.TableName(), field.DBName, table) + } + } + }) + } +} diff --git a/backend/plugins/schema_e2e/migration_upgrade_path_test.go b/backend/plugins/schema_e2e/migration_upgrade_path_test.go new file mode 100644 index 00000000000..60c8de40ffd --- /dev/null +++ b/backend/plugins/schema_e2e/migration_upgrade_path_test.go @@ -0,0 +1,382 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package schema_e2e + +import ( + "fmt" + "testing" + + "github.com/apache/incubator-devlake/core/config" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/impls/dalgorm" + "github.com/apache/incubator-devlake/impls/logruslog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" + + copilotimpl "github.com/apache/incubator-devlake/plugins/gh-copilot/impl" + jiraimpl "github.com/apache/incubator-devlake/plugins/jira/impl" + taigaimpl "github.com/apache/incubator-devlake/plugins/taiga/impl" + teambitionimpl "github.com/apache/incubator-devlake/plugins/teambition/impl" + testmoimpl "github.com/apache/incubator-devlake/plugins/testmo/impl" +) + +// ---------------------------------------------------------------------------- +// Pre-repair table shapes +// +// Each struct reproduces a table EXACTLY as the buggy migration left it, i.e. +// without the columns that the repair migration adds. The repair script is then +// executed against that table *with rows in it*. +// ---------------------------------------------------------------------------- + +// upgradePreJiraSprintReport is _tool_jira_sprint_reports as created by +// 20260722 — no embedded NoPKModel, hence no _raw_data_* / created_at / +// updated_at columns. +type upgradePreJiraSprintReport struct { + ConnectionId uint64 `gorm:"primaryKey"` + BoardId uint64 `gorm:"primaryKey"` + SprintId uint64 `gorm:"primaryKey"` + IssueId uint64 `gorm:"primaryKey"` + + IssueKey string `gorm:"type:varchar(255)"` + Bucket string `gorm:"type:varchar(32);index"` + Done bool + StoryPointsAtSprintStart *float64 + StoryPointsAtSprintEnd *float64 +} + +func (upgradePreJiraSprintReport) TableName() string { return "_tool_jira_sprint_reports" } + +// upgradePreTaigaScopeConfig lacks `type_mappings`. +type upgradePreTaigaScopeConfig struct { + archived.Model + Entities []string `gorm:"type:json;serializer:json"` + ConnectionId uint64 `gorm:"index"` + Name string `gorm:"type:varchar(255);uniqueIndex"` +} + +func (upgradePreTaigaScopeConfig) TableName() string { return "_tool_taiga_scope_configs" } + +// upgradePreTeambitionScopeConfig lacks the embedded common.Model, i.e. the +// table has no primary key at all and no id / created_at / updated_at. +type upgradePreTeambitionScopeConfig struct { + Entities []string `gorm:"type:json;serializer:json"` + ConnectionId uint64 `gorm:"index"` + Name string `gorm:"type:varchar(255)"` + TypeMappings map[string]string `gorm:"serializer:json"` + StatusMappings map[string]string `gorm:"serializer:json"` + BugDueDateField string `gorm:"column:bug_due_date_field"` + TaskDueDateField string `gorm:"column:task_due_date_field"` + StoryDueDateField string `gorm:"column:story_due_date_field"` +} + +func (upgradePreTeambitionScopeConfig) TableName() string { + return "_tool_teambition_scope_configs" +} + +// upgradePreTestmoScopeConfig lacks `connection_id` and `name`. +type upgradePreTestmoScopeConfig struct { + archived.Model + Entities []string `gorm:"type:json;serializer:json"` + AcceptanceTestPattern string `gorm:"type:varchar(255)"` + SmokeTestPattern string `gorm:"type:varchar(255)"` + TeamPattern string `gorm:"type:varchar(255)"` +} + +func (upgradePreTestmoScopeConfig) TableName() string { return "_tool_testmo_scope_configs" } + +// upgradePreCopilotEnterpriseCredits and friends lack the seven credit +// breakdown columns, which 20260708 declared through an unexported embedded +// struct that GORM silently ignores. +type upgradePreCopilotEnterpriseCredits struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(191)"` + Year int `gorm:"primaryKey"` + Month int `gorm:"primaryKey"` + Day int `gorm:"primaryKey"` + Enterprise string `gorm:"primaryKey;type:varchar(191)"` + Model string `gorm:"primaryKey;type:varchar(191)"` + Product string `gorm:"type:varchar(32)"` + archived.NoPKModel +} + +func (upgradePreCopilotEnterpriseCredits) TableName() string { + return "_tool_copilot_enterprise_ai_credit_usage" +} + +type upgradePreCopilotOrgCredits struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(191)"` + Year int `gorm:"primaryKey"` + Month int `gorm:"primaryKey"` + Day int `gorm:"primaryKey"` + Organization string `gorm:"primaryKey;type:varchar(191)"` + Model string `gorm:"primaryKey;type:varchar(191)"` + Product string `gorm:"type:varchar(32)"` + archived.NoPKModel +} + +func (upgradePreCopilotOrgCredits) TableName() string { + return "_tool_copilot_org_ai_credit_usage" +} + +type upgradePreCopilotUserCredits struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(191)"` + Year int `gorm:"primaryKey"` + Month int `gorm:"primaryKey"` + Day int `gorm:"primaryKey"` + User string `gorm:"primaryKey;type:varchar(191)"` + Model string `gorm:"primaryKey;type:varchar(191)"` + Product string `gorm:"type:varchar(32)"` + archived.NoPKModel +} + +func (upgradePreCopilotUserCredits) TableName() string { + return "_tool_copilot_user_ai_credit_usage" +} + +var copilotCreditColumns = []string{ + "gross_quantity", "discount_quantity", "net_quantity", "price_per_unit", + "gross_amount", "discount_amount", "net_amount", +} + +// upgradeCase describes one repair migration and how to exercise it on a table +// that already contains data. +type upgradeCase struct { + // plugin owning the migration script, used to look the script up by version + // instead of duplicating it here (so the test breaks if the script is + // removed or renumbered). + plugin plugin.PluginMeta + version uint64 + // pre is the table as the buggy migration left it. + pre dal.Tabler + // seed rows inserted BEFORE the repair migration runs. + seed []map[string]interface{} + // wantColumns must exist after the repair. + wantColumns []string + // wantPrimaryKey asserts the table has a primary key afterwards. A plain + // AutoMigrate cannot add one, which is invisible to a column-only check + // (and silently accepted by PostgreSQL). + wantPrimaryKey bool + // autoIncColumn, if set, must be backfilled with distinct non-zero values + // for the pre-existing rows, and a subsequent INSERT must still work. + autoIncColumn string +} + +// TestMigrationUpgradePathOnPopulatedTables complements +// TestMigrationSchemaMatchesModels: that guard proves the END STATE of a fresh +// migration run matches the models, but every table it inspects is empty, so it +// cannot exercise the upgrade path of a repair migration on a database that +// already holds rows — which is the only situation those migrations exist for. +// +// For each repair migration this test therefore +// 1. recreates the table exactly as the buggy migration left it, +// 2. inserts rows, +// 3. runs ONLY that repair script, +// 4. asserts the columns were added, the rows survived untouched, the primary +// key exists and auto-increment ids were backfilled. +// +// Step 4 is what a column-presence check on an empty table cannot see: replacing +// the explicit AUTO_INCREMENT DDL in the teambition script with a plain +// AutoMigrate is accepted by PostgreSQL (it adds `bigserial` without a primary +// key), and only this test notices. +func TestMigrationUpgradePathOnPopulatedTables(t *testing.T) { + db := e2ehelper.NewIsolatedMigrationDb(t, "upgrade_path") + dalInstance := dalgorm.NewDalgorm(db) + basicRes := runner.CreateBasicRes(config.GetConfig(), logruslog.Global, db) + + cases := map[string]upgradeCase{ + "jira sprint report raw data columns": { + plugin: jiraimpl.Jira{}, + version: 20260727000000, + pre: upgradePreJiraSprintReport{}, + seed: []map[string]interface{}{ + {"connection_id": 1, "board_id": 10, "sprint_id": 100, "issue_id": 1000, "issue_key": "TEST-1", "bucket": "committed", "done": false}, + {"connection_id": 1, "board_id": 10, "sprint_id": 100, "issue_id": 1001, "issue_key": "TEST-2", "bucket": "completed", "done": true}, + }, + wantColumns: []string{"_raw_data_params", "_raw_data_table", "_raw_data_id", "_raw_data_remark", "created_at", "updated_at"}, + }, + "taiga scope config type_mappings": { + plugin: taigaimpl.Taiga{}, + version: 20260727000001, + pre: upgradePreTaigaScopeConfig{}, + seed: []map[string]interface{}{ + {"id": 1, "connection_id": 1, "name": "taiga-cfg-a"}, + {"id": 2, "connection_id": 1, "name": "taiga-cfg-b"}, + }, + wantColumns: []string{"type_mappings"}, + wantPrimaryKey: true, + }, + "teambition scope config primary key": { + plugin: teambitionimpl.Teambition{}, + version: 20260727000001, + pre: upgradePreTeambitionScopeConfig{}, + seed: []map[string]interface{}{ + {"connection_id": 1, "name": "teambition-cfg-a"}, + {"connection_id": 1, "name": "teambition-cfg-b"}, + }, + wantColumns: []string{"id", "created_at", "updated_at"}, + wantPrimaryKey: true, + autoIncColumn: "id", + }, + "testmo scope config connection_id/name": { + plugin: testmoimpl.Testmo{}, + version: 20260727000001, + pre: upgradePreTestmoScopeConfig{}, + seed: []map[string]interface{}{ + // `name` carries a uniqueIndex in the repaired shape; both rows + // are backfilled with NULL, which MySQL and PostgreSQL accept. + {"id": 1, "acceptance_test_pattern": "a"}, + {"id": 2, "acceptance_test_pattern": "b"}, + }, + wantColumns: []string{"connection_id", "name"}, + wantPrimaryKey: true, + }, + "gh-copilot enterprise credit breakdown": { + plugin: copilotimpl.GhCopilot{}, + version: 20260731000000, + pre: upgradePreCopilotEnterpriseCredits{}, + seed: []map[string]interface{}{ + {"connection_id": 1, "scope_id": "ent-1", "year": 2026, "month": 8, "day": 1, "enterprise": "acme", "model": "gpt-4.1", "product": "copilot"}, + }, + wantColumns: copilotCreditColumns, + }, + "gh-copilot org credit breakdown": { + plugin: copilotimpl.GhCopilot{}, + version: 20260731000000, + pre: upgradePreCopilotOrgCredits{}, + seed: []map[string]interface{}{ + {"connection_id": 1, "scope_id": "org-1", "year": 2026, "month": 8, "day": 1, "organization": "acme", "model": "gpt-4.1", "product": "copilot"}, + }, + wantColumns: copilotCreditColumns, + }, + "gh-copilot user credit breakdown": { + plugin: copilotimpl.GhCopilot{}, + version: 20260731000000, + pre: upgradePreCopilotUserCredits{}, + seed: []map[string]interface{}{ + {"connection_id": 1, "scope_id": "user-1", "year": 2026, "month": 8, "day": 1, "user": "octocat", "model": "gpt-4.1", "product": "copilot"}, + }, + wantColumns: copilotCreditColumns, + }, + } + + for name, c := range cases { + c := c + t.Run(name, func(t *testing.T) { + table := c.pre.TableName() + script := findMigrationScript(t, c.plugin, c.version) + + // 1. table exactly as the buggy migration left it + require.NoError(t, db.Migrator().DropTable(table)) + require.NoError(t, db.Table(table).AutoMigrate(c.pre)) + for _, column := range c.wantColumns { + require.Falsef(t, db.Migrator().HasColumn(c.pre, column), + "precondition failed: %q already has column %q, the pre-repair shape is wrong", + table, column) + } + + // 2. rows, so the migration has to survive real data + for _, row := range c.seed { + require.NoError(t, db.Table(table).Create(row).Error) + } + + // 3. run ONLY the repair script + require.NoErrorf(t, script.Up(basicRes), + "migration %q failed on a populated %q", script.Name(), table) + + // 4a. the columns are there now + actual, colErr := dal.GetColumnNames(dalInstance, dal.DefaultTabler{Name: table}, + func(dal.ColumnMeta) bool { return true }) + require.NoError(t, colErr) + existing := make(map[string]struct{}, len(actual)) + for _, column := range actual { + existing[column] = struct{}{} + } + for _, column := range c.wantColumns { + _, ok := existing[column] + assert.Truef(t, ok, "table %q is still missing column %q after %q", + table, column, script.Name()) + } + + // 4b. no data was lost + var rows int64 + require.NoError(t, db.Table(table).Count(&rows).Error) + assert.EqualValuesf(t, len(c.seed), rows, + "migration %q changed the row count of %q", script.Name(), table) + + // 4c. the primary key survived / was created + if c.wantPrimaryKey { + assert.Truef(t, hasPrimaryKey(t, db, table), + "table %q has no primary key after %q — AutoMigrate cannot add one, "+ + "the script needs explicit DDL", table, script.Name()) + } + + // 4d. auto-increment ids were backfilled and the counter still works + if c.autoIncColumn != "" { + var ids []uint64 + require.NoError(t, db.Table(table).Pluck(c.autoIncColumn, &ids).Error) + require.Len(t, ids, len(c.seed)) + seen := map[uint64]struct{}{} + for _, id := range ids { + assert.NotZerof(t, id, "pre-existing row was not assigned a %q", c.autoIncColumn) + _, dup := seen[id] + assert.Falsef(t, dup, "duplicate %q=%d after backfill", c.autoIncColumn, id) + seen[id] = struct{}{} + } + require.NoErrorf(t, db.Table(table).Create(map[string]interface{}{ + "connection_id": 2, "name": "inserted-after-migration", + }).Error, "INSERT after the migration failed, the sequence/counter is out of sync") + } + }) + } +} + +// findMigrationScript looks the script up through the plugin's own +// MigrationScripts() so the test fails if it is removed or renumbered. +func findMigrationScript(t *testing.T, p plugin.PluginMeta, version uint64) plugin.MigrationScript { + migratable, ok := p.(plugin.PluginMigration) + require.Truef(t, ok, "plugin %s does not implement PluginMigration", p.Name()) + for _, script := range migratable.MigrationScripts() { + if script.Version() == version { + return script + } + } + t.Fatalf("plugin %s has no migration script with version %d", p.Name(), version) + return nil +} + +// hasPrimaryKey works on MySQL and PostgreSQL alike. +func hasPrimaryKey(t *testing.T, db *gorm.DB, table string) bool { + schemaFunc := "current_schema()" + if db.Dialector.Name() == "mysql" { + schemaFunc = "DATABASE()" + } + var count int64 + err := db.Raw(fmt.Sprintf( + `SELECT COUNT(*) FROM information_schema.table_constraints + WHERE constraint_type = 'PRIMARY KEY' AND table_name = ? AND table_schema = %s`, + schemaFunc), table).Scan(&count).Error + require.NoError(t, err) + return count > 0 +} diff --git a/backend/plugins/sonarqube/api/connection_api.go b/backend/plugins/sonarqube/api/connection_api.go index feb1905a63d..d387e5d460a 100644 --- a/backend/plugins/sonarqube/api/connection_api.go +++ b/backend/plugins/sonarqube/api/connection_api.go @@ -44,6 +44,9 @@ func testConnection(ctx context.Context, connection models.SonarqubeConn) (*plug return nil, errors.Default.Wrap(err, "error validating target") } } + if err := connection.ValidateUserTokenPrefix(); err != nil { + return nil, err + } apiClient, err := api.NewApiClientFromConnection(ctx, basicRes, &connection) if err != nil { return nil, err diff --git a/backend/plugins/sonarqube/e2e/issue_code_block_long_component_test.go b/backend/plugins/sonarqube/e2e/issue_code_block_long_component_test.go new file mode 100644 index 00000000000..e1a7d52b5a5 --- /dev/null +++ b/backend/plugins/sonarqube/e2e/issue_code_block_long_component_test.go @@ -0,0 +1,205 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "strings" + "testing" + "time" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/codequality" + coremigrations "github.com/apache/incubator-devlake/core/models/migrationscripts" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + implcontext "github.com/apache/incubator-devlake/impls/context" + "github.com/apache/incubator-devlake/plugins/sonarqube/impl" + "github.com/apache/incubator-devlake/plugins/sonarqube/models" + sonarqubemigrations "github.com/apache/incubator-devlake/plugins/sonarqube/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/sonarqube/tasks" + "github.com/stretchr/testify/require" +) + +type sonarqubeIssueCodeBlockBeforeText struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey"` + IssueKey string `gorm:"index"` + Component string `gorm:"index;type:varchar(500)"` + StartLine int + EndLine int + StartOffset int + EndOffset int + Msg string + common.NoPKModel +} + +func (sonarqubeIssueCodeBlockBeforeText) TableName() string { + return "_tool_sonarqube_issue_code_blocks" +} + +type cqIssueCodeBlockBeforeText struct { + domainlayer.DomainEntity + IssueKey string `json:"key" gorm:"index"` + Component string `gorm:"index"` + StartLine int + EndLine int + StartOffset int + EndOffset int + Msg string +} + +func (cqIssueCodeBlockBeforeText) TableName() string { + return "cq_issue_code_blocks" +} + +func TestSonarqubeIssueCodeBlockLongComponent(t *testing.T) { + var sonarqube impl.Sonarqube + dataflowTester := e2ehelper.NewDataFlowTester(t, "sonarqube", sonarqube) + dataflowTester.FlushTabler(&models.SonarqubeIssue{}) + require.NoError(t, dataflowTester.Db.Migrator().DropTable( + &sonarqubeIssueCodeBlockBeforeText{}, + &cqIssueCodeBlockBeforeText{}, + )) + require.NoError(t, dataflowTester.Db.AutoMigrate( + &sonarqubeIssueCodeBlockBeforeText{}, + &cqIssueCodeBlockBeforeText{}, + )) + require.NoError(t, dataflowTester.Db.Migrator().DropIndex( + "cq_issue_code_blocks", "idx_cq_issue_code_blocks_component", + )) + require.NoError(t, dataflowTester.Db.Migrator().DropIndex( + "_tool_sonarqube_issue_code_blocks", "idx__tool_sonarqube_issue_code_blocks_component", + )) + + existingComponent := "existing:component" + require.NoError(t, dataflowTester.Db.Create(&sonarqubeIssueCodeBlockBeforeText{ + ConnectionId: 1, + Id: "existing-tool-block", + IssueKey: "existing-issue", + Component: existingComponent, + }).Error) + require.NoError(t, dataflowTester.Db.Create(&cqIssueCodeBlockBeforeText{ + DomainEntity: domainlayer.DomainEntity{Id: "existing-domain-block"}, + IssueKey: "existing-domain-issue", + Component: existingComponent, + }).Error) + + basicRes := implcontext.NewDefaultBasicRes(dataflowTester.Cfg, dataflowTester.Log, dataflowTester.Dal) + runMigration(t, coremigrations.All(), "change cq_issue_code_blocks.component type to text", basicRes) + runMigration(t, sonarqubemigrations.All(), "change _tool_sonarqube_issue_code_blocks.component type to text", basicRes) + assertTextColumnWithoutIndex(t, dataflowTester, "cq_issue_code_blocks") + assertTextColumnWithoutIndex(t, dataflowTester, "_tool_sonarqube_issue_code_blocks") + + var migratedToolBlock sonarqubeIssueCodeBlockBeforeText + require.NoError(t, dataflowTester.Db.First(&migratedToolBlock, "id = ?", "existing-tool-block").Error) + require.Equal(t, existingComponent, migratedToolBlock.Component) + var migratedDomainBlock cqIssueCodeBlockBeforeText + require.NoError(t, dataflowTester.Db.First(&migratedDomainBlock, "id = ?", "existing-domain-block").Error) + require.Equal(t, existingComponent, migratedDomainBlock.Component) + require.NoError(t, dataflowTester.Db.Delete(&migratedToolBlock).Error) + require.NoError(t, dataflowTester.Db.Delete(&migratedDomainBlock).Error) + + longComponent256 := "project:" + strings.Repeat("a", 256) + longComponent500 := "project:" + strings.Repeat("b", 500) + require.Greater(t, len(longComponent256), 256) + require.Greater(t, len(longComponent500), 500) + + issueKey := "TEST-LONG-COMPONENT-ISSUE" + projectKey := "test-long-component-project" + result := dataflowTester.Db.Create(&models.SonarqubeIssue{ + ConnectionId: 1, + IssueKey: issueKey, + ProjectKey: projectKey, + Component: longComponent500, + Rule: "java:S3776", + Severity: "CRITICAL", + }) + require.NoError(t, result.Error) + + codeBlocks := []*models.SonarqubeIssueCodeBlock{ + { + ConnectionId: 1, + Id: "test-long-component-block-256", + IssueKey: issueKey, + Component: longComponent256, + Msg: "component longer than 256 characters", + }, + { + ConnectionId: 1, + Id: "test-long-component-block-500", + IssueKey: issueKey, + Component: longComponent500, + Msg: "component longer than 500 characters", + }, + } + for _, block := range codeBlocks { + require.NoError(t, dataflowTester.Db.Create(block).Error) + } + + dataflowTester.Subtask(tasks.ConvertIssueCodeBlocksMeta, &tasks.SonarqubeTaskData{ + Options: &tasks.SonarqubeOptions{ + ConnectionId: 1, + ProjectKey: projectKey, + }, + TaskStartTime: time.Now(), + }) + + var domainBlocks []codequality.CqIssueCodeBlock + require.NoError(t, dataflowTester.Db.Find(&domainBlocks).Error) + require.Len(t, domainBlocks, 2) + require.ElementsMatch(t, + []string{longComponent256, longComponent500}, + []string{domainBlocks[0].Component, domainBlocks[1].Component}, + ) + + var toolBlocks []models.SonarqubeIssueCodeBlock + require.NoError(t, dataflowTester.Db.Where( + "connection_id = ? AND issue_key = ?", 1, issueKey, + ).Find(&toolBlocks).Error) + require.Len(t, toolBlocks, 2) + require.ElementsMatch(t, + []string{longComponent256, longComponent500}, + []string{toolBlocks[0].Component, toolBlocks[1].Component}, + ) +} + +func runMigration(t *testing.T, scripts []plugin.MigrationScript, name string, basicRes *implcontext.DefaultBasicRes) { + t.Helper() + for _, script := range scripts { + if script.Name() == name { + require.NoError(t, script.Up(basicRes)) + return + } + } + require.Fail(t, "migration is not registered", name) +} + +func assertTextColumnWithoutIndex(t *testing.T, dataflowTester *e2ehelper.DataFlowTester, table string) { + t.Helper() + columnTypes, err := dataflowTester.Db.Migrator().ColumnTypes(table) + require.NoError(t, err) + for _, columnType := range columnTypes { + if columnType.Name() == "component" { + require.Contains(t, strings.ToLower(columnType.DatabaseTypeName()), "text") + require.False(t, dataflowTester.Db.Migrator().HasIndex(table, "idx_"+table+"_component")) + return + } + } + require.Fail(t, "component column not found", table) +} diff --git a/backend/plugins/sonarqube/impl/impl.go b/backend/plugins/sonarqube/impl/impl.go index a6d6bd258ab..85ee668ee31 100644 --- a/backend/plugins/sonarqube/impl/impl.go +++ b/backend/plugins/sonarqube/impl/impl.go @@ -86,6 +86,8 @@ func (p Sonarqube) GetTablesInfo() []dal.Tabler { &models.SonarqubeFileMetrics{}, &models.SonarqubeAccount{}, &models.SonarqubeScopeConfig{}, + &models.SonarqubeProjectMetricsHistory{}, + &models.SonarqubeProjectAnalysis{}, } } @@ -108,6 +110,11 @@ func (p Sonarqube) SubTaskMetas() []plugin.SubTaskMeta { tasks.ConvertHotspotsMeta, tasks.ConvertFileMetricsMeta, tasks.ConvertAccountsMeta, + tasks.CollectProjectMetricsHistoryMeta, + tasks.ExtractProjectMetricsHistoryMeta, + tasks.CollectProjectAnalysesMeta, + tasks.ExtractProjectAnalysesMeta, + tasks.ConvertProjectMetricsHistoryMeta, } } diff --git a/backend/plugins/sonarqube/models/connection.go b/backend/plugins/sonarqube/models/connection.go index 66c32f57c49..0bbe124d592 100644 --- a/backend/plugins/sonarqube/models/connection.go +++ b/backend/plugins/sonarqube/models/connection.go @@ -21,6 +21,7 @@ import ( "encoding/base64" "fmt" "net/http" + "strings" "github.com/apache/incubator-devlake/core/utils" @@ -29,6 +30,12 @@ import ( helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" ) +const ( + userTokenPrefix = "squ_" + globalAnalysisTokenPrefix = "sqa_" + projectAnalysisTokenPrefix = "sqp_" +) + type SonarqubeAccessToken helper.AccessToken // SetupAuthentication sets up the HTTP Request Authentication @@ -92,7 +99,40 @@ func (connection *SonarqubeConnection) MergeFromRequest(target *SonarqubeConnect return nil } -func (connection *SonarqubeConnection) IsCloud() bool { +// ValidateUserTokenPrefix ensures the token is a SonarQube User token on Server +// instances. Global (sqa_) and Project (sqp_) analysis tokens authenticate but +// cannot call read APIs such as measures/component_tree. +func (connection SonarqubeConn) ValidateUserTokenPrefix() errors.Error { + if connection.IsCloud() { + return nil + } + token := strings.TrimSpace(connection.Token) + if token == "" { + return errors.BadInput.New("token is required") + } + switch { + case strings.HasPrefix(token, globalAnalysisTokenPrefix): + return errors.BadInput.New( + "DevLake requires a User token (squ_ prefix). " + + "Global Analysis tokens (sqa_) can push scan results but cannot read project metrics via the Web API. " + + "Create a User token under My Account > Security in SonarQube.", + ) + case strings.HasPrefix(token, projectAnalysisTokenPrefix): + return errors.BadInput.New( + "DevLake requires a User token (squ_ prefix). " + + "Project Analysis tokens (sqp_) can push scan results but cannot read project metrics via the Web API. " + + "Create a User token under My Account > Security in SonarQube.", + ) + case !strings.HasPrefix(token, userTokenPrefix): + return errors.BadInput.New( + "DevLake requires a User token (squ_ prefix) for SonarQube Server. " + + "Create one under My Account > Security in SonarQube.", + ) + } + return nil +} + +func (connection SonarqubeConn) IsCloud() bool { return connection.Endpoint == "https://sonarcloud.io/api/" } diff --git a/backend/plugins/sonarqube/models/connection_test.go b/backend/plugins/sonarqube/models/connection_test.go new file mode 100644 index 00000000000..e92284ad5da --- /dev/null +++ b/backend/plugins/sonarqube/models/connection_test.go @@ -0,0 +1,92 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "testing" + + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +func TestValidateUserTokenPrefix(t *testing.T) { + t.Parallel() + + serverEndpoint := "https://rad-sonar.example.com/api/" + cloudEndpoint := "https://sonarcloud.io/api/" + + tests := []struct { + name string + conn SonarqubeConn + wantErr bool + }{ + { + name: "user token on server", + conn: SonarqubeConn{ + RestConnection: helper.RestConnection{Endpoint: serverEndpoint}, + SonarqubeAccessToken: SonarqubeAccessToken{Token: "squ_abc123"}, + }, + }, + { + name: "global analysis token on server", + conn: SonarqubeConn{ + RestConnection: helper.RestConnection{Endpoint: serverEndpoint}, + SonarqubeAccessToken: SonarqubeAccessToken{Token: "sqa_abc123"}, + }, + wantErr: true, + }, + { + name: "project analysis token on server", + conn: SonarqubeConn{ + RestConnection: helper.RestConnection{Endpoint: serverEndpoint}, + SonarqubeAccessToken: SonarqubeAccessToken{Token: "sqp_abc123"}, + }, + wantErr: true, + }, + { + name: "unknown prefix on server", + conn: SonarqubeConn{ + RestConnection: helper.RestConnection{Endpoint: serverEndpoint}, + SonarqubeAccessToken: SonarqubeAccessToken{Token: "legacy-token"}, + }, + wantErr: true, + }, + { + name: "sonarcloud skips prefix check", + conn: SonarqubeConn{ + RestConnection: helper.RestConnection{Endpoint: cloudEndpoint}, + SonarqubeAccessToken: SonarqubeAccessToken{Token: "sqa_abc123"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + err := tt.conn.ValidateUserTokenPrefix() + if tt.wantErr { + if err == nil { + t.Fatal("expected error, got nil") + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + } +} diff --git a/backend/plugins/sonarqube/models/migrationscripts/20260701_change_issue_code_block_component_type.go b/backend/plugins/sonarqube/models/migrationscripts/20260701_change_issue_code_block_component_type.go new file mode 100644 index 00000000000..3ba3a00eb2e --- /dev/null +++ b/backend/plugins/sonarqube/models/migrationscripts/20260701_change_issue_code_block_component_type.go @@ -0,0 +1,44 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*changeIssueCodeBlockComponentType)(nil) + +type changeIssueCodeBlockComponentType struct{} + +func (script *changeIssueCodeBlockComponentType) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + if err := db.DropIndexes("_tool_sonarqube_issue_code_blocks", "idx__tool_sonarqube_issue_code_blocks_component"); err != nil { + return err + } + return db.ModifyColumnType("_tool_sonarqube_issue_code_blocks", "component", "text") +} + +func (*changeIssueCodeBlockComponentType) Version() uint64 { + return 20260701000000 +} + +func (*changeIssueCodeBlockComponentType) Name() string { + return "change _tool_sonarqube_issue_code_blocks.component type to text" +} diff --git a/backend/plugins/sonarqube/models/migrationscripts/20260707_add_project_metrics_history.go b/backend/plugins/sonarqube/models/migrationscripts/20260707_add_project_metrics_history.go new file mode 100644 index 00000000000..d060f46310a --- /dev/null +++ b/backend/plugins/sonarqube/models/migrationscripts/20260707_add_project_metrics_history.go @@ -0,0 +1,77 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +var _ plugin.MigrationScript = (*addProjectMetricsHistory)(nil) + +type projectMetricsHistory20260707 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ProjectKey string `gorm:"primaryKey;type:varchar(255)"` + AnalysisDate time.Time `gorm:"primaryKey"` + MetricKey string `gorm:"primaryKey;type:varchar(100)"` + MetricValue string `gorm:"type:varchar(50)"` + archived.NoPKModel +} + +func (projectMetricsHistory20260707) TableName() string { + return "_tool_sonarqube_project_metrics_history" +} + +type projectAnalyses20260707 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ProjectKey string `gorm:"primaryKey;type:varchar(255)"` + AnalysisKey string `gorm:"primaryKey;type:varchar(255)"` + AnalysisDate time.Time `gorm:"index"` + ProjectVersion string `gorm:"type:varchar(255)"` + Revision string `gorm:"type:varchar(255)"` + BuildString string `gorm:"type:varchar(255)"` + DetectedCI string `gorm:"type:varchar(100)"` + archived.NoPKModel +} + +func (projectAnalyses20260707) TableName() string { + return "_tool_sonarqube_project_analyses" +} + +type addProjectMetricsHistory struct{} + +func (script *addProjectMetricsHistory) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &projectMetricsHistory20260707{}, + &projectAnalyses20260707{}, + ) +} + +func (*addProjectMetricsHistory) Version() uint64 { + return 20260707153200 +} + +func (*addProjectMetricsHistory) Name() string { + return "add project_metrics_history and project_analyses tables" +} diff --git a/backend/plugins/sonarqube/models/migrationscripts/register.go b/backend/plugins/sonarqube/models/migrationscripts/register.go index 7c48de84226..4e1e590b9fa 100644 --- a/backend/plugins/sonarqube/models/migrationscripts/register.go +++ b/backend/plugins/sonarqube/models/migrationscripts/register.go @@ -39,5 +39,7 @@ func All() []plugin.MigrationScript { new(addOrgToConn), new(addIssueImpacts), new(extendSonarqubeFieldSize), + new(changeIssueCodeBlockComponentType), + new(addProjectMetricsHistory), } } diff --git a/backend/plugins/sonarqube/models/sonarqube_issue_code_block.go b/backend/plugins/sonarqube/models/sonarqube_issue_code_block.go index b5bb58bddae..b36386338da 100644 --- a/backend/plugins/sonarqube/models/sonarqube_issue_code_block.go +++ b/backend/plugins/sonarqube/models/sonarqube_issue_code_block.go @@ -23,7 +23,7 @@ type SonarqubeIssueCodeBlock struct { ConnectionId uint64 `gorm:"primaryKey"` Id string `gorm:"primaryKey"` IssueKey string `gorm:"index"` - Component string `gorm:"index;type:varchar(500)"` + Component string `gorm:"type:text"` StartLine int EndLine int StartOffset int diff --git a/backend/plugins/sonarqube/models/sonarqube_project_analyses.go b/backend/plugins/sonarqube/models/sonarqube_project_analyses.go new file mode 100644 index 00000000000..55dac5ff702 --- /dev/null +++ b/backend/plugins/sonarqube/models/sonarqube_project_analyses.go @@ -0,0 +1,40 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +type SonarqubeProjectAnalysis struct { + ConnectionId uint64 `gorm:"primaryKey"` + ProjectKey string `gorm:"primaryKey;type:varchar(255)"` + AnalysisKey string `gorm:"primaryKey;type:varchar(255)"` + AnalysisDate time.Time `gorm:"index"` + ProjectVersion string `gorm:"type:varchar(255)"` + Revision string `gorm:"type:varchar(255)"` + BuildString string `gorm:"type:varchar(255)"` + DetectedCI string `gorm:"type:varchar(100)"` + common.NoPKModel +} + +func (SonarqubeProjectAnalysis) TableName() string { + return "_tool_sonarqube_project_analyses" +} diff --git a/backend/plugins/sonarqube/models/sonarqube_project_metrics_history.go b/backend/plugins/sonarqube/models/sonarqube_project_metrics_history.go new file mode 100644 index 00000000000..ebd555252ac --- /dev/null +++ b/backend/plugins/sonarqube/models/sonarqube_project_metrics_history.go @@ -0,0 +1,37 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +type SonarqubeProjectMetricsHistory struct { + ConnectionId uint64 `gorm:"primaryKey"` + ProjectKey string `gorm:"primaryKey;type:varchar(255)"` + AnalysisDate time.Time `gorm:"primaryKey"` + MetricKey string `gorm:"primaryKey;type:varchar(100)"` + MetricValue string `gorm:"type:varchar(50)"` + common.NoPKModel +} + +func (SonarqubeProjectMetricsHistory) TableName() string { + return "_tool_sonarqube_project_metrics_history" +} diff --git a/backend/plugins/sonarqube/tasks/issue_code_blocks_convertor_test.go b/backend/plugins/sonarqube/tasks/issue_code_blocks_convertor_test.go new file mode 100644 index 00000000000..c3d703fe2a0 --- /dev/null +++ b/backend/plugins/sonarqube/tasks/issue_code_blocks_convertor_test.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + "testing" + + "github.com/apache/incubator-devlake/core/models/domainlayer/codequality" + "github.com/apache/incubator-devlake/plugins/sonarqube/models" + "github.com/stretchr/testify/require" +) + +func TestIssueCodeBlockComponentIsText(t *testing.T) { + testCases := []struct { + name string + model interface{} + }{ + {name: "tool layer", model: models.SonarqubeIssueCodeBlock{}}, + {name: "domain layer", model: codequality.CqIssueCodeBlock{}}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + field, ok := reflect.TypeOf(testCase.model).FieldByName("Component") + require.True(t, ok) + require.Equal(t, "type:text", field.Tag.Get("gorm")) + }) + } +} diff --git a/backend/plugins/sonarqube/tasks/project_analyses_collector.go b/backend/plugins/sonarqube/tasks/project_analyses_collector.go new file mode 100644 index 00000000000..37682fd7cd0 --- /dev/null +++ b/backend/plugins/sonarqube/tasks/project_analyses_collector.go @@ -0,0 +1,92 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/sonarqube/models" +) + +const RAW_PROJECT_ANALYSES_TABLE = "sonarqube_api_project_analyses" + +var _ plugin.SubTaskEntryPoint = CollectProjectAnalyses + +func CollectProjectAnalyses(taskCtx plugin.SubTaskContext) errors.Error { + logger := taskCtx.GetLogger() + logger.Info("collect project analyses") + + data := taskCtx.GetData().(*SonarqubeTaskData) + apiCollector, err := helper.NewStatefulApiCollector(helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.SonarqubeApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectKey: data.Options.ProjectKey, + }, + Table: RAW_PROJECT_ANALYSES_TABLE, + }) + if err != nil { + return err + } + + err = apiCollector.InitCollector(helper.ApiCollectorArgs{ + ApiClient: data.ApiClient, + PageSize: 500, + UrlTemplate: "project_analyses/search", + Query: func(reqData *helper.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("project", data.Options.ProjectKey) + query.Set("ps", fmt.Sprintf("%v", reqData.Pager.Size)) + query.Set("p", fmt.Sprintf("%v", reqData.Pager.Page)) + if apiCollector.GetSince() != nil { + query.Set("from", apiCollector.GetSince().UTC().Format("2006-01-02")) + } + return query, nil + }, + GetTotalPages: GetTotalPagesFromResponse, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var resData struct { + Analyses []json.RawMessage `json:"analyses"` + } + err := helper.UnmarshalResponse(res, &resData) + if err != nil { + return nil, err + } + return resData.Analyses, nil + }, + }) + if err != nil { + return err + } + + return apiCollector.Execute() +} + +var CollectProjectAnalysesMeta = plugin.SubTaskMeta{ + Name: "CollectProjectAnalyses", + EntryPoint: CollectProjectAnalyses, + EnabledByDefault: true, + Description: "Collect project analysis metadata from SonarQube project_analyses/search API", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_QUALITY}, +} diff --git a/backend/plugins/sonarqube/tasks/project_analyses_extractor.go b/backend/plugins/sonarqube/tasks/project_analyses_extractor.go new file mode 100644 index 00000000000..ac10d8d6e09 --- /dev/null +++ b/backend/plugins/sonarqube/tasks/project_analyses_extractor.go @@ -0,0 +1,84 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/sonarqube/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractProjectAnalyses + +type projectAnalysisResponse struct { + Key string `json:"key"` + Date string `json:"date"` + ProjectVersion string `json:"projectVersion"` + Revision string `json:"revision"` + BuildString string `json:"buildString"` + DetectedCI string `json:"detectedCI"` +} + +func ExtractProjectAnalyses(taskCtx plugin.SubTaskContext) errors.Error { + rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PROJECT_ANALYSES_TABLE) + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: *rawDataSubTaskArgs, + Extract: func(resData *helper.RawData) ([]interface{}, errors.Error) { + body := &projectAnalysisResponse{} + err := errors.Convert(json.Unmarshal(resData.Data, body)) + if err != nil { + return nil, err + } + + analysisDate, parseErr := time.Parse("2006-01-02T15:04:05-0700", body.Date) + if parseErr != nil { + return nil, errors.Default.Wrap(errors.Convert(parseErr), "failed to parse analysis date") + } + + analysis := &models.SonarqubeProjectAnalysis{ + ConnectionId: data.Options.ConnectionId, + ProjectKey: data.Options.ProjectKey, + AnalysisKey: body.Key, + AnalysisDate: analysisDate, + ProjectVersion: body.ProjectVersion, + Revision: body.Revision, + BuildString: body.BuildString, + DetectedCI: body.DetectedCI, + } + return []interface{}{analysis}, nil + }, + }) + if err != nil { + return err + } + + return extractor.Execute() +} + +var ExtractProjectAnalysesMeta = plugin.SubTaskMeta{ + Name: "ExtractProjectAnalyses", + EntryPoint: ExtractProjectAnalyses, + EnabledByDefault: true, + Description: "Extract raw project analyses data into tool layer table", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_QUALITY}, +} diff --git a/backend/plugins/sonarqube/tasks/project_metrics_history_collector.go b/backend/plugins/sonarqube/tasks/project_metrics_history_collector.go new file mode 100644 index 00000000000..fa2b02633b6 --- /dev/null +++ b/backend/plugins/sonarqube/tasks/project_metrics_history_collector.go @@ -0,0 +1,94 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/sonarqube/models" +) + +const RAW_PROJECT_METRICS_HISTORY_TABLE = "sonarqube_api_project_metrics_history" + +const metricsToCollect = "coverage,ncloc,bugs,vulnerabilities,code_smells,security_hotspots," + + "duplicated_lines_density,sqale_rating,reliability_rating,security_rating,complexity,cognitive_complexity" + +var _ plugin.SubTaskEntryPoint = CollectProjectMetricsHistory + +func CollectProjectMetricsHistory(taskCtx plugin.SubTaskContext) errors.Error { + logger := taskCtx.GetLogger() + logger.Info("collect project metrics history") + + data := taskCtx.GetData().(*SonarqubeTaskData) + apiCollector, err := helper.NewStatefulApiCollector(helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.SonarqubeApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectKey: data.Options.ProjectKey, + }, + Table: RAW_PROJECT_METRICS_HISTORY_TABLE, + }) + if err != nil { + return err + } + + err = apiCollector.InitCollector(helper.ApiCollectorArgs{ + ApiClient: data.ApiClient, + PageSize: 1000, + UrlTemplate: "measures/search_history", + Query: func(reqData *helper.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("component", data.Options.ProjectKey) + query.Set("metrics", metricsToCollect) + query.Set("ps", fmt.Sprintf("%v", reqData.Pager.Size)) + query.Set("p", fmt.Sprintf("%v", reqData.Pager.Page)) + if apiCollector.GetSince() != nil { + query.Set("from", apiCollector.GetSince().UTC().Format("2006-01-02")) + } + return query, nil + }, + GetTotalPages: GetTotalPagesFromResponse, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var body json.RawMessage + err := helper.UnmarshalResponse(res, &body) + if err != nil { + return nil, err + } + return []json.RawMessage{body}, nil + }, + }) + if err != nil { + return err + } + + return apiCollector.Execute() +} + +var CollectProjectMetricsHistoryMeta = plugin.SubTaskMeta{ + Name: "CollectProjectMetricsHistory", + EntryPoint: CollectProjectMetricsHistory, + EnabledByDefault: true, + Description: "Collect project-level metric history from SonarQube measures/search_history API", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_QUALITY}, +} diff --git a/backend/plugins/sonarqube/tasks/project_metrics_history_convertor.go b/backend/plugins/sonarqube/tasks/project_metrics_history_convertor.go new file mode 100644 index 00000000000..bf59aa1fe98 --- /dev/null +++ b/backend/plugins/sonarqube/tasks/project_metrics_history_convertor.go @@ -0,0 +1,158 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "reflect" + "strconv" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/codequality" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + sonarqubeModels "github.com/apache/incubator-devlake/plugins/sonarqube/models" +) + +var ConvertProjectMetricsHistoryMeta = plugin.SubTaskMeta{ + Name: "convertProjectMetricsHistory", + EntryPoint: ConvertProjectMetricsHistory, + EnabledByDefault: true, + Description: "Convert tool layer project metrics history into domain layer table cq_project_metrics_history", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_QUALITY}, +} + +func ConvertProjectMetricsHistory(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + _, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PROJECT_METRICS_HISTORY_TABLE) + + // Query all narrow metric rows ordered so we can group-pivot them + cursor, err := db.Cursor( + dal.From(sonarqubeModels.SonarqubeProjectMetricsHistory{}), + dal.Where("connection_id = ? AND project_key = ?", data.Options.ConnectionId, data.Options.ProjectKey), + dal.Orderby("analysis_date"), + ) + if err != nil { + return err + } + defer cursor.Close() + + projectIdGen := didgen.NewDomainIdGenerator(&sonarqubeModels.SonarqubeProject{}) + domainProjectKey := projectIdGen.Generate(data.Options.ConnectionId, data.Options.ProjectKey) + + batchSave, err := helper.NewBatchSave(taskCtx, reflect.TypeOf(&codequality.CqProjectMetricsHistory{}), 200) + if err != nil { + return err + } + defer batchSave.Close() + + // Group narrow rows by analysis_date, pivot into wide domain rows + var currentDate *time.Time + var currentDomain *codequality.CqProjectMetricsHistory + + flushCurrent := func() errors.Error { + if currentDomain != nil { + return batchSave.Add(currentDomain) + } + return nil + } + + for cursor.Next() { + row := &sonarqubeModels.SonarqubeProjectMetricsHistory{} + err = db.Fetch(cursor, row) + if err != nil { + return err + } + + if currentDate == nil || !currentDate.Equal(row.AnalysisDate) { + if flushErr := flushCurrent(); flushErr != nil { + return flushErr + } + domainId := fmt.Sprintf("%s:%s", + domainProjectKey, + row.AnalysisDate.UTC().Format("2006-01-02T15:04:05Z"), + ) + currentDomain = &codequality.CqProjectMetricsHistory{ + DomainEntity: domainlayer.DomainEntity{Id: domainId}, + ProjectKey: domainProjectKey, + AnalysisDate: row.AnalysisDate, + } + t := row.AnalysisDate + currentDate = &t + } + + applyMetricValue(currentDomain, row.MetricKey, row.MetricValue) + } + + if flushErr := flushCurrent(); flushErr != nil { + return flushErr + } + + return batchSave.Close() +} + +func applyMetricValue(d *codequality.CqProjectMetricsHistory, metricKey, value string) { + switch metricKey { + case "coverage": + if v, err := strconv.ParseFloat(value, 64); err == nil { + d.Coverage = &v + } + case "ncloc": + if v, err := strconv.Atoi(value); err == nil { + d.Ncloc = &v + } + case "bugs": + if v, err := strconv.Atoi(value); err == nil { + d.Bugs = &v + } + case "reliability_rating": + d.ReliabilityRating = alphabetMap[value] + case "code_smells": + if v, err := strconv.Atoi(value); err == nil { + d.CodeSmells = &v + } + case "sqale_rating": + d.SqaleRating = alphabetMap[value] + case "complexity": + if v, err := strconv.Atoi(value); err == nil { + d.Complexity = &v + } + case "cognitive_complexity": + if v, err := strconv.Atoi(value); err == nil { + d.CognitiveComplexity = &v + } + case "vulnerabilities": + if v, err := strconv.Atoi(value); err == nil { + d.Vulnerabilities = &v + } + case "security_rating": + d.SecurityRating = alphabetMap[value] + case "security_hotspots": + if v, err := strconv.Atoi(value); err == nil { + d.SecurityHotspots = &v + } + case "duplicated_lines_density": + if v, err := strconv.ParseFloat(value, 64); err == nil { + d.DuplicatedLinesDensity = &v + } + } +} diff --git a/backend/plugins/sonarqube/tasks/project_metrics_history_extractor.go b/backend/plugins/sonarqube/tasks/project_metrics_history_extractor.go new file mode 100644 index 00000000000..430ff196beb --- /dev/null +++ b/backend/plugins/sonarqube/tasks/project_metrics_history_extractor.go @@ -0,0 +1,89 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/sonarqube/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractProjectMetricsHistory + +type metricsHistoryResponse struct { + Measures []struct { + Metric string `json:"metric"` + History []struct { + Date string `json:"date"` + Value string `json:"value"` + } `json:"history"` + } `json:"measures"` +} + +func ExtractProjectMetricsHistory(taskCtx plugin.SubTaskContext) errors.Error { + rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PROJECT_METRICS_HISTORY_TABLE) + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: *rawDataSubTaskArgs, + Extract: func(resData *helper.RawData) ([]interface{}, errors.Error) { + body := &metricsHistoryResponse{} + err := errors.Convert(json.Unmarshal(resData.Data, body)) + if err != nil { + return nil, err + } + + var results []interface{} + for _, measure := range body.Measures { + for _, entry := range measure.History { + if entry.Value == "" { + continue + } + analysisDate, parseErr := time.Parse("2006-01-02T15:04:05-0700", entry.Date) + if parseErr != nil { + return nil, errors.Default.Wrap(errors.Convert(parseErr), "failed to parse analysis date") + } + results = append(results, &models.SonarqubeProjectMetricsHistory{ + ConnectionId: data.Options.ConnectionId, + ProjectKey: data.Options.ProjectKey, + AnalysisDate: analysisDate, + MetricKey: measure.Metric, + MetricValue: entry.Value, + }) + } + } + return results, nil + }, + }) + if err != nil { + return err + } + + return extractor.Execute() +} + +var ExtractProjectMetricsHistoryMeta = plugin.SubTaskMeta{ + Name: "ExtractProjectMetricsHistory", + EntryPoint: ExtractProjectMetricsHistory, + EnabledByDefault: true, + Description: "Extract raw project metrics history into tool layer table", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_QUALITY}, +} diff --git a/backend/plugins/table_info_test.go b/backend/plugins/table_info_test.go index 0d4482ca6c9..c3262153dfc 100644 --- a/backend/plugins/table_info_test.go +++ b/backend/plugins/table_info_test.go @@ -30,6 +30,7 @@ import ( bitbucket_server "github.com/apache/incubator-devlake/plugins/bitbucket_server/impl" circleci "github.com/apache/incubator-devlake/plugins/circleci/impl" claudeCode "github.com/apache/incubator-devlake/plugins/claude_code/impl" + clickup "github.com/apache/incubator-devlake/plugins/clickup/impl" customize "github.com/apache/incubator-devlake/plugins/customize/impl" dbt "github.com/apache/incubator-devlake/plugins/dbt/impl" dora "github.com/apache/incubator-devlake/plugins/dora/impl" @@ -41,6 +42,7 @@ import ( githubGraphql "github.com/apache/incubator-devlake/plugins/github_graphql/impl" gitlab "github.com/apache/incubator-devlake/plugins/gitlab/impl" icla "github.com/apache/incubator-devlake/plugins/icla/impl" + incidentio "github.com/apache/incubator-devlake/plugins/incidentio/impl" issueTrace "github.com/apache/incubator-devlake/plugins/issue_trace/impl" jenkins "github.com/apache/incubator-devlake/plugins/jenkins/impl" jira "github.com/apache/incubator-devlake/plugins/jira/impl" @@ -87,6 +89,7 @@ func Test_GetPluginTablesInfo(t *testing.T) { checker.FeedIn("github_graphql", githubGraphql.GithubGraphql{}.GetTablesInfo) checker.FeedIn("gitlab/models", gitlab.Gitlab{}.GetTablesInfo) checker.FeedIn("icla/models", icla.Icla{}.GetTablesInfo) + checker.FeedIn("incidentio/models", incidentio.Incidentio{}.GetTablesInfo) checker.FeedIn("jenkins/models", jenkins.Jenkins{}.GetTablesInfo) checker.FeedIn("jira/models", jira.Jira{}.GetTablesInfo) checker.FeedIn("linear/models", linear.Linear{}.GetTablesInfo) @@ -107,6 +110,7 @@ func Test_GetPluginTablesInfo(t *testing.T) { checker.FeedIn("zentao/models", zentao.Zentao{}.GetTablesInfo) checker.FeedIn("claude_code/models", claudeCode.ClaudeCode{}.GetTablesInfo) checker.FeedIn("circleci/models", circleci.Circleci{}.GetTablesInfo) + checker.FeedIn("clickup/models", clickup.ClickUp{}.GetTablesInfo) checker.FeedIn("opsgenie/models", opsgenie.Opsgenie{}.GetTablesInfo) checker.FeedIn("linker/models", linker.Linker{}.GetTablesInfo) checker.FeedIn("issue_trace/models", issueTrace.IssueTrace{}.GetTablesInfo) diff --git a/backend/plugins/taiga/models/migrationscripts/20260727_add_missing_scope_config_columns.go b/backend/plugins/taiga/models/migrationscripts/20260727_add_missing_scope_config_columns.go new file mode 100644 index 00000000000..84fa66cda41 --- /dev/null +++ b/backend/plugins/taiga/models/migrationscripts/20260727_add_missing_scope_config_columns.go @@ -0,0 +1,61 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +// taigaScopeConfig20260727 mirrors models.TaigaScopeConfig. The initial +// migration created `_tool_taiga_scope_configs` without the `type_mappings` +// column, while the runtime model declares it — every read/write of the model +// would fail with "Unknown column 'type_mappings'". +// +// The `uniqueIndex` on `name` is safe to add here: the new column is nullable, +// so pre-existing rows are backfilled with NULL, and both MySQL and PostgreSQL +// allow duplicate NULLs in a unique index (verified against both engines). +type taigaScopeConfig20260727 struct { + archived.Model + Entities []string `gorm:"type:json;serializer:json" json:"entities"` + ConnectionId uint64 `json:"connectionId" gorm:"index"` + Name string `json:"name" gorm:"type:varchar(255);uniqueIndex"` + TypeMappings map[string]json.RawMessage `json:"typeMappings" gorm:"type:json;serializer:json"` +} + +func (taigaScopeConfig20260727) TableName() string { + return "_tool_taiga_scope_configs" +} + +type addMissingScopeConfigColumns struct{} + +func (script *addMissingScopeConfigColumns) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, &taigaScopeConfig20260727{}) +} + +func (*addMissingScopeConfigColumns) Version() uint64 { + return 20260727000001 +} + +func (*addMissingScopeConfigColumns) Name() string { + return "add missing type_mappings column to _tool_taiga_scope_configs" +} diff --git a/backend/plugins/taiga/models/migrationscripts/register.go b/backend/plugins/taiga/models/migrationscripts/register.go index d2cfd08e269..da91884aa1f 100644 --- a/backend/plugins/taiga/models/migrationscripts/register.go +++ b/backend/plugins/taiga/models/migrationscripts/register.go @@ -26,5 +26,6 @@ func All() []plugin.MigrationScript { return []plugin.MigrationScript{ new(addInitTables20250220), new(addTaskIssueEpicTables20260306), + new(addMissingScopeConfigColumns), } } diff --git a/backend/plugins/teambition/models/migrationscripts/20260727_add_missing_scope_config_columns.go b/backend/plugins/teambition/models/migrationscripts/20260727_add_missing_scope_config_columns.go new file mode 100644 index 00000000000..0b29b9ca044 --- /dev/null +++ b/backend/plugins/teambition/models/migrationscripts/20260727_add_missing_scope_config_columns.go @@ -0,0 +1,90 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "fmt" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +const teambitionScopeConfigTable20260727 = "_tool_teambition_scope_configs" + +// teambitionScopeConfig20260727 mirrors models.TeambitionScopeConfig. The +// migration that created `_tool_teambition_scope_configs` did not include the +// columns of the embedded common.Model (`id`, `created_at`, `updated_at`), +// which the runtime model expects. +type teambitionScopeConfig20260727 struct { + archived.Model + Entities []string `gorm:"type:json;serializer:json" json:"entities"` + ConnectionId uint64 `json:"connectionId" gorm:"index"` + Name string `json:"name" gorm:"type:varchar(255);uniqueIndex"` + TypeMappings map[string]string `json:"typeMappings" gorm:"serializer:json"` + StatusMappings map[string]string `json:"statusMappings" gorm:"serializer:json"` + BugDueDateField string `json:"bugDueDateField" gorm:"column:bug_due_date_field"` + TaskDueDateField string `json:"taskDueDateField" gorm:"column:task_due_date_field"` + StoryDueDateField string `json:"storyDueDateField" gorm:"column:story_due_date_field"` +} + +func (teambitionScopeConfig20260727) TableName() string { + return teambitionScopeConfigTable20260727 +} + +type addMissingScopeConfigColumns struct{} + +// Up adds the columns of the embedded common.Model that the runtime model +// expects. +// +// `id` is an auto-increment primary key, which GORM's AutoMigrate cannot append +// to an existing table: it emits a plain `ADD COLUMN ... AUTO_INCREMENT`, which +// MySQL rejects with "Incorrect table definition; there can be only one auto +// column and it must be defined as a key". The column is therefore added with +// explicit DDL (the table has no primary key so far), letting the database +// backfill ids for existing rows and keep the sequence/counter in sync. The +// remaining columns (`created_at`, `updated_at`) and the indexes are then +// created by AutoMigrate as usual. +func (script *addMissingScopeConfigColumns) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + if !db.HasColumn(teambitionScopeConfigTable20260727, "id") { + ddl := fmt.Sprintf( + "ALTER TABLE %s ADD COLUMN id BIGSERIAL PRIMARY KEY", + teambitionScopeConfigTable20260727, + ) + if db.Dialect() == "mysql" { + ddl = fmt.Sprintf( + "ALTER TABLE %s ADD COLUMN id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY", + teambitionScopeConfigTable20260727, + ) + } + if err := db.Exec(ddl); err != nil { + return err + } + } + return migrationhelper.AutoMigrateTables(basicRes, &teambitionScopeConfig20260727{}) +} + +func (*addMissingScopeConfigColumns) Version() uint64 { + return 20260727000001 +} + +func (*addMissingScopeConfigColumns) Name() string { + return "add missing id/created_at/updated_at columns to _tool_teambition_scope_configs" +} diff --git a/backend/plugins/teambition/models/migrationscripts/register.go b/backend/plugins/teambition/models/migrationscripts/register.go index f9914e7a12c..d761a15fb70 100644 --- a/backend/plugins/teambition/models/migrationscripts/register.go +++ b/backend/plugins/teambition/models/migrationscripts/register.go @@ -26,5 +26,6 @@ func All() []plugin.MigrationScript { new(reCreateTeambitionConnections), new(addScopeConfigId), new(addAppIdBack), + new(addMissingScopeConfigColumns), } } diff --git a/backend/plugins/testmo/models/migrationscripts/20260727_add_missing_scope_config_columns.go b/backend/plugins/testmo/models/migrationscripts/20260727_add_missing_scope_config_columns.go new file mode 100644 index 00000000000..11048f60a74 --- /dev/null +++ b/backend/plugins/testmo/models/migrationscripts/20260727_add_missing_scope_config_columns.go @@ -0,0 +1,61 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +// testmoScopeConfig20260727 mirrors models.TestmoScopeConfig. The migration +// that created `_tool_testmo_scope_configs` omitted the `connection_id` and +// `name` columns of the embedded common.ScopeConfig, which the runtime model +// expects. +// +// The `uniqueIndex` on `name` is safe to add here: the new column is nullable, +// so pre-existing rows are backfilled with NULL, and both MySQL and PostgreSQL +// allow duplicate NULLs in a unique index (verified against both engines). +type testmoScopeConfig20260727 struct { + archived.Model + Entities []string `gorm:"type:json;serializer:json" json:"entities"` + ConnectionId uint64 `json:"connectionId" gorm:"index"` + Name string `json:"name" gorm:"type:varchar(255);uniqueIndex"` + AcceptanceTestPattern string `json:"acceptanceTestPattern" gorm:"type:varchar(255)"` + SmokeTestPattern string `json:"smokeTestPattern" gorm:"type:varchar(255)"` + TeamPattern string `json:"teamPattern" gorm:"type:varchar(255)"` +} + +func (testmoScopeConfig20260727) TableName() string { + return "_tool_testmo_scope_configs" +} + +type addMissingScopeConfigColumns struct{} + +func (script *addMissingScopeConfigColumns) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, &testmoScopeConfig20260727{}) +} + +func (*addMissingScopeConfigColumns) Version() uint64 { + return 20260727000001 +} + +func (*addMissingScopeConfigColumns) Name() string { + return "add missing connection_id/name columns to _tool_testmo_scope_configs" +} diff --git a/backend/plugins/testmo/models/migrationscripts/register.go b/backend/plugins/testmo/models/migrationscripts/register.go index 7843d4b84e1..95f2ae48b41 100644 --- a/backend/plugins/testmo/models/migrationscripts/register.go +++ b/backend/plugins/testmo/models/migrationscripts/register.go @@ -25,5 +25,6 @@ func All() []plugin.MigrationScript { new(addScopeConfigIdToProjects), new(replaceTestsWithRuns), new(fixRawTableNamesAndSchemas), + new(addMissingScopeConfigColumns), } } diff --git a/backend/plugins/zentao/e2e/snapshot_tables/execution_sprint.csv b/backend/plugins/zentao/e2e/snapshot_tables/execution_sprint.csv index adcc379f389..9568cddeca0 100644 --- a/backend/plugins/zentao/e2e/snapshot_tables/execution_sprint.csv +++ b/backend/plugins/zentao/e2e/snapshot_tables/execution_sprint.csv @@ -1,9 +1,9 @@ -id,name,url,status,started_date,ended_date,completed_date,original_board_id -zentao:ZentaoExecution:1:133,为长期项目创建一个迭代,https://zentaomax.demo.qucheng.cc/execution-view-133.html,,2023-08-28T00:00:00.000+00:00,2023-09-08T00:00:00.000+00:00,,zentao:ZentaoProject:1:192 -zentao:ZentaoExecution:1:193,为长期项目创建一个迭代,https://zentaomax.demo.qucheng.cc/execution-view-193.html,ACTIVE,2023-08-28T00:00:00.000+00:00,2023-09-08T00:00:00.000+00:00,,zentao:ZentaoProject:1:192 -zentao:ZentaoExecution:1:194,新建一个迭代，完成之后检查sprints表,https://zentaomax.demo.qucheng.cc/execution-view-194.html,CLOSED,2023-08-29T00:00:00.000+00:00,2023-09-05T00:00:00.000+00:00,2023-08-29T08:04:13.000+00:00,zentao:ZentaoProject:1:192 -zentao:ZentaoExecution:1:266,新建迭代然后关闭,https://zentaomax.demo.qucheng.cc/execution-view-266.html,CLOSED,2023-09-07T00:00:00.000+00:00,2023-09-13T00:00:00.000+00:00,2023-09-07T02:15:31.000+00:00,zentao:ZentaoProject:1:192 -zentao:ZentaoExecution:1:267,建一个计划未来开始迭代,https://zentaomax.demo.qucheng.cc/execution-view-267.html,FUTURE,2023-09-09T00:00:00.000+00:00,2023-09-22T00:00:00.000+00:00,,zentao:ZentaoProject:1:192 -zentao:ZentaoExecution:1:268,起始-截止都是结束时间,https://zentaomax.demo.qucheng.cc/execution-view-268.html,FUTURE,2023-09-04T00:00:00.000+00:00,2023-09-05T00:00:00.000+00:00,,zentao:ZentaoProject:1:192 -zentao:ZentaoExecution:1:269,把迭代挂起,https://zentaomax.demo.qucheng.cc/execution-view-269.html,SUSPENDED,2023-09-07T00:00:00.000+00:00,2023-09-20T00:00:00.000+00:00,,zentao:ZentaoProject:1:192 -zentao:ZentaoExecution:1:270,流程扭转--挂起迭代2,https://zentaomax.demo.qucheng.cc/execution-view-270.html,SUSPENDED,2023-09-07T00:00:00.000+00:00,2023-09-20T00:00:00.000+00:00,,zentao:ZentaoProject:1:192 +id,name,url,status,started_date,ended_date,completed_date,original_board_id,committed_story_point,completed_story_point +zentao:ZentaoExecution:1:133,为长期项目创建一个迭代,https://zentaomax.demo.qucheng.cc/execution-view-133.html,,2023-08-28T00:00:00.000+00:00,2023-09-08T00:00:00.000+00:00,,zentao:ZentaoProject:1:192,, +zentao:ZentaoExecution:1:193,为长期项目创建一个迭代,https://zentaomax.demo.qucheng.cc/execution-view-193.html,ACTIVE,2023-08-28T00:00:00.000+00:00,2023-09-08T00:00:00.000+00:00,,zentao:ZentaoProject:1:192,, +zentao:ZentaoExecution:1:194,新建一个迭代，完成之后检查sprints表,https://zentaomax.demo.qucheng.cc/execution-view-194.html,CLOSED,2023-08-29T00:00:00.000+00:00,2023-09-05T00:00:00.000+00:00,2023-08-29T08:04:13.000+00:00,zentao:ZentaoProject:1:192,, +zentao:ZentaoExecution:1:266,新建迭代然后关闭,https://zentaomax.demo.qucheng.cc/execution-view-266.html,CLOSED,2023-09-07T00:00:00.000+00:00,2023-09-13T00:00:00.000+00:00,2023-09-07T02:15:31.000+00:00,zentao:ZentaoProject:1:192,, +zentao:ZentaoExecution:1:267,建一个计划未来开始迭代,https://zentaomax.demo.qucheng.cc/execution-view-267.html,FUTURE,2023-09-09T00:00:00.000+00:00,2023-09-22T00:00:00.000+00:00,,zentao:ZentaoProject:1:192,, +zentao:ZentaoExecution:1:268,起始-截止都是结束时间,https://zentaomax.demo.qucheng.cc/execution-view-268.html,FUTURE,2023-09-04T00:00:00.000+00:00,2023-09-05T00:00:00.000+00:00,,zentao:ZentaoProject:1:192,, +zentao:ZentaoExecution:1:269,把迭代挂起,https://zentaomax.demo.qucheng.cc/execution-view-269.html,SUSPENDED,2023-09-07T00:00:00.000+00:00,2023-09-20T00:00:00.000+00:00,,zentao:ZentaoProject:1:192,, +zentao:ZentaoExecution:1:270,流程扭转--挂起迭代2,https://zentaomax.demo.qucheng.cc/execution-view-270.html,SUSPENDED,2023-09-07T00:00:00.000+00:00,2023-09-20T00:00:00.000+00:00,,zentao:ZentaoProject:1:192,, \ No newline at end of file diff --git a/backend/python/DevelopmentSetup.md b/backend/python/DevelopmentSetup.md index 4bc885a96a3..c831313c8de 100644 --- a/backend/python/DevelopmentSetup.md +++ b/backend/python/DevelopmentSetup.md @@ -22,12 +22,12 @@ limitations under the License. For the Python plugin tests to run properly, the following steps must be taken: 1. The following packages are required for Ubuntu: `libffi-dev default-libmysqlclient-dev libpq-dev` -2. `python3.9` is required by the time of this document. - - Try `deadsnakes` if you are using Ubuntu 22.04 or above, the `python3.9-dev` is required. - - Use `virtualenv` if you are having multiple python versions. `virtualenv -p python3.9 path/to/venv` and `source path/to/venv/bin/activate.sh` should do the trick -3. both `mysql-client` and `postgresql` are required. +2. `python3.11` is required by the time of this document. + - Try `deadsnakes` if you are using Ubuntu 22.04 or above, the `python3.11-dev` is required. + - Use `virtualenv` if you are having multiple python versions. `virtualenv -p python3.11 path/to/venv` and `source path/to/venv/bin/activate.sh` should do the trick +3. both `mysql-client` and `postgresql` are required. - `postgresql` is required for `psycopg2` to work -4. [poetry](https://python-poetry.org/) is required. +4. [poetry](https://python-poetry.org/) is required. - run `cd backend/python/pydevlake && poetry install` - run `cd backend/python/plugins/azuredevops && poetry install` 5. `sqlalchemy` won't work with `localhost` in the database connection string, use `127.0.0.1` instead diff --git a/backend/python/plugins/azuredevops/pyproject.toml b/backend/python/plugins/azuredevops/pyproject.toml index df7e8d2552d..74ee730f87b 100644 --- a/backend/python/plugins/azuredevops/pyproject.toml +++ b/backend/python/plugins/azuredevops/pyproject.toml @@ -21,7 +21,7 @@ authors = ["Hezheng Yin "] readme = "README.md" [tool.poetry.dependencies] -python = "~3.9" +python = ">=3.11,<3.12" pydevlake = { path = "../../pydevlake", develop = true } diff --git a/backend/python/pydevlake/pydevlake/domain_layer/code.py b/backend/python/pydevlake/pydevlake/domain_layer/code.py index cffe7f50297..bd28802c907 100644 --- a/backend/python/pydevlake/pydevlake/domain_layer/code.py +++ b/backend/python/pydevlake/pydevlake/domain_layer/code.py @@ -50,7 +50,7 @@ class PullRequest(DomainModel, table=True): class PullRequestLabels(NoPKModel, table=True): __tablename__ = 'pull_request_labels' pull_request_id: str = Field(primary_key=True) - label_name: str + label_name: str = Field(primary_key=True) class PullRequestCommit(NoPKModel, table=True): @@ -94,7 +94,7 @@ class Commit(NoPKModel, table=True): class CommitParent(NoPKModel, table=True): __tablename__ = 'commit_parents' commit_sha: str = Field(primary_key=True) - parent_commit_sha: str + parent_commit_sha: str = Field(primary_key=True) class CommitsDiff(DomainModel, table=True): @@ -112,6 +112,7 @@ class RefCommit(NoPKModel, table=True): new_commit_sha: str old_commit_sha: str + class Component(NoPKModel, table=True): __tablename__ = 'components' repo_id: str diff --git a/backend/python/pydevlake/pyproject.toml b/backend/python/pydevlake/pyproject.toml index 9f52bf821a5..c18cd9f62b3 100644 --- a/backend/python/pydevlake/pyproject.toml +++ b/backend/python/pydevlake/pyproject.toml @@ -22,7 +22,7 @@ license = "Apache-2.0" readme = "README.md" [tool.poetry.dependencies] -python = "~3.9" +python = ">=3.11,<3.12" sqlmodel = "^0.0.8" mysqlclient = "^2.1.1" requests = "^2.28.1" diff --git a/backend/python/test/fakeplugin/pyproject.toml b/backend/python/test/fakeplugin/pyproject.toml index 9f17ea3b30b..d72afa4f9b3 100644 --- a/backend/python/test/fakeplugin/pyproject.toml +++ b/backend/python/test/fakeplugin/pyproject.toml @@ -20,7 +20,7 @@ description = "Fake python plugin used only in tests" authors = [] [tool.poetry.dependencies] -python = "~3.9" +python = ">=3.11,<3.12" pydevlake = { path = "../../pydevlake", develop = true } diff --git a/backend/python/uv.sh b/backend/python/uv.sh index 9e9d7696f18..b775b9787c0 100755 --- a/backend/python/uv.sh +++ b/backend/python/uv.sh @@ -50,7 +50,7 @@ sync_project() { rm -rf .venv fi if [ ! -x .venv/bin/python ]; then - uv venv --python "${DEVLAKE_PYTHON_VERSION:-3.9}" .venv + uv venv --python "${DEVLAKE_PYTHON_VERSION:-3.11}" .venv fi uv pip install --python .venv/bin/python -e . } diff --git a/backend/scripts/build-plugins.sh b/backend/scripts/build-plugins.sh index 03fb7b4f9fb..e0ef94bb9b7 100755 --- a/backend/scripts/build-plugins.sh +++ b/backend/scripts/build-plugins.sh @@ -52,7 +52,8 @@ fi if [ -z "$DEVLAKE_PLUGINS" ]; then echo "Building all plugins" - PLUGINS=$(find $PLUGIN_SRC_DIR/* -maxdepth 0 -type d -not -name core -not -name helper -not -name logs -not -empty) + # schema_e2e is not a plugin, it only holds the cross-plugin schema-drift e2e test + PLUGINS=$(find $PLUGIN_SRC_DIR/* -maxdepth 0 -type d -not -name core -not -name helper -not -name logs -not -name schema_e2e -not -empty) else echo "Building the following plugins: $PLUGIN" PLUGINS= diff --git a/backend/server/api/auth/auth.go b/backend/server/api/auth/auth.go index ea7029a3824..aead556a5bc 100644 --- a/backend/server/api/auth/auth.go +++ b/backend/server/api/auth/auth.go @@ -308,6 +308,12 @@ func (s *Service) Callback(c *gin.Context) { fail(c, http.StatusBadGateway, "extract claims", err) return } + + if !s.cfg.IsUserAllowed(email) { + fail(c, http.StatusForbidden, "user is not allowed", nil) + return + } + jti := uuid.NewString() jwt, expiresAt, err := oidchelper.IssueSession(s.cfg, jti, state.Provider, sub, email, name) if err != nil { diff --git a/backend/server/api/middlewares.go b/backend/server/api/middlewares.go index a988c41288a..e1609b1ec6a 100644 --- a/backend/server/api/middlewares.go +++ b/backend/server/api/middlewares.go @@ -33,6 +33,7 @@ import ( "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/helpers/apikeyhelper" + "github.com/apache/incubator-devlake/server/api/shared" "github.com/gin-gonic/gin" ) @@ -272,9 +273,13 @@ func CheckAuthorizationHeader(c *gin.Context, logger log.Logger, db dal.Dal, api logger.Info("redirect path: %s to: %s", c.Request.URL.Path, path) c.Request.URL.Path = path - c.Set(common.USER, &common.User{ + user := &common.User{ Name: apiKey.Creator.Creator, Email: apiKey.Creator.CreatorEmail, - }) + } + c.Set(common.USER, user) + // Also store in the request context so the user survives gin's HandleContext + // resetting c.Keys when rerouting from /rest/... to /plugins/... + c.Request = shared.SetRestAuthUser(c.Request, user) return true } diff --git a/backend/server/api/middlewares_rest_auth_test.go b/backend/server/api/middlewares_rest_auth_test.go new file mode 100644 index 00000000000..41d421b6529 --- /dev/null +++ b/backend/server/api/middlewares_rest_auth_test.go @@ -0,0 +1,107 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/apache/incubator-devlake/core/config" + coremodels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/helpers/apikeyhelper" + contextimpl "github.com/apache/incubator-devlake/impls/context" + "github.com/apache/incubator-devlake/impls/logruslog" + mockdal "github.com/apache/incubator-devlake/mocks/core/dal" + "github.com/apache/incubator-devlake/server/api/shared" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/mock" +) + +// requireUserGate simulates RequireAuth with AUTH_ENABLED=true: it rejects any +// request whose gin context does not carry an authenticated user. This is the +// exact check that caused 401s for valid REST API key requests. +func requireUserGate() gin.HandlerFunc { + return func(c *gin.Context) { + if _, ok := shared.GetUser(c); !ok { + c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{ + "success": false, + "message": "unauthorized", + }) + return + } + c.Next() + } +} + +// TestRestAuthKeyReachesHandlerWhenAuthEnabled sends a valid /rest/... Bearer +// token request through a router that also has a RequireAuth-style gate. The +// request must reach the downstream handler (200) rather than being rejected +// by the auth gate (401). +// +// Without the fix, gin's HandleContext resets c.Keys, wiping the user that +// RestAuthentication stored before rerouting, so the auth gate sees no user +// and returns 401. +func TestRestAuthKeyReachesHandlerWhenAuthEnabled(t *testing.T) { + gin.SetMode(gin.TestMode) + + // apikeyhelper reads ENCRYPTION_SECRET from the global viper config. + const encryptionSecret = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" // 32 bytes + config.GetConfig().Set("ENCRYPTION_SECRET", encryptionSecret) + + basicRes := contextimpl.NewDefaultBasicRes(config.GetConfig(), logruslog.Global, nil) + helper := apikeyhelper.NewApiKeyHelper(basicRes, logruslog.Global) + + const plaintext = "test-api-key-plaintext" + hashedKey, hashErr := helper.DigestToken(plaintext) + if hashErr != nil { + t.Fatalf("DigestToken: %v", hashErr) + } + + // Mock DAL: when First is called, populate the destination with a valid key + // whose AllowedPath covers the webhook endpoint under test. + db := &mockdal.Dal{} + db.On("First", mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + dst := args.Get(0).(*coremodels.ApiKey) + dst.ApiKey = hashedKey + dst.AllowedPath = `/plugins/webhook/connections/1/.*` + dst.Creator = common.Creator{Creator: "test-user"} + }). + Return(nil) + + basicRes = contextimpl.NewDefaultBasicRes(config.GetConfig(), logruslog.Global, db) + + router := gin.New() + router.Use(RestAuthentication(router, basicRes)) + router.Use(requireUserGate()) + router.POST("/plugins/webhook/connections/:id/deployments", func(c *gin.Context) { + c.Status(http.StatusOK) + }) + + req := httptest.NewRequest(http.MethodPost, "/rest/plugins/webhook/connections/1/deployments", nil) + req.Header.Set("Authorization", "Bearer "+plaintext) + resp := httptest.NewRecorder() + router.ServeHTTP(resp, req) + + if resp.Code != http.StatusOK { + t.Fatalf("expected 200 with valid REST API key when auth gate is active, got %d: %s", + resp.Code, resp.Body.String()) + } +} diff --git a/backend/server/api/shared/gin_utils.go b/backend/server/api/shared/gin_utils.go index 892dc489ae6..c15d0a73019 100644 --- a/backend/server/api/shared/gin_utils.go +++ b/backend/server/api/shared/gin_utils.go @@ -18,15 +18,36 @@ limitations under the License. package shared import ( + "context" + "net/http" + "github.com/apache/incubator-devlake/core/models/common" "github.com/gin-gonic/gin" ) +// restAuthKey is an unexported type used as a request-context key so it cannot +// collide with keys set by other packages. +type restAuthKey struct{} + +// SetRestAuthUser stores the authenticated user in the HTTP request context. +// This is necessary because gin's HandleContext calls c.reset(), which clears +// c.Keys but leaves c.Request (and its context) intact. RestAuthentication +// calls this before rerouting so the user survives the reset. +func SetRestAuthUser(r *http.Request, user *common.User) *http.Request { + return r.WithContext(context.WithValue(r.Context(), restAuthKey{}, user)) +} + func GetUser(c *gin.Context) (*common.User, bool) { userObj, exist := c.Get(common.USER) - if !exist { - return nil, false + if exist { + if user, ok := userObj.(*common.User); ok { + return user, true + } + } + // Fallback: RestAuthentication stores the user here before calling + // HandleContext, which resets c.Keys but preserves c.Request. + if user, ok := c.Request.Context().Value(restAuthKey{}).(*common.User); ok && user != nil { + return user, true } - user := userObj.(*common.User) - return user, true + return nil, false } diff --git a/backend/test/e2e/remote/docker-compose.test.yml b/backend/test/e2e/remote/docker-compose.test.yml index 393cab00a3c..63827d73c26 100644 --- a/backend/test/e2e/remote/docker-compose.test.yml +++ b/backend/test/e2e/remote/docker-compose.test.yml @@ -19,7 +19,7 @@ version: "3" services: mysql-test: - image: mysql:8.0.26 + image: mysql:8.4.10 platform: linux/x86_64 volumes: - mysql-test-storage:/var/lib/mysql @@ -33,12 +33,12 @@ services: MYSQL_PASSWORD: merico postgres-test: - image: postgres:14.2-alpine + image: postgres:18.1-alpine restart: always ports: - "3308:5432" volumes: - - postgres-test-storage:/var/lib/postgresql/data + - postgres-test-storage:/var/lib/postgresql environment: POSTGRES_DB: lake POSTGRES_USER: merico diff --git a/backend/test/e2e/services/server_startup_test.go b/backend/test/e2e/services/server_startup_test.go index 9432f80c263..f52dafeb038 100644 --- a/backend/test/e2e/services/server_startup_test.go +++ b/backend/test/e2e/services/server_startup_test.go @@ -34,6 +34,7 @@ import ( githubGraphql "github.com/apache/incubator-devlake/plugins/github_graphql/impl" gitlab "github.com/apache/incubator-devlake/plugins/gitlab/impl" icla "github.com/apache/incubator-devlake/plugins/icla/impl" + incidentio "github.com/apache/incubator-devlake/plugins/incidentio/impl" jenkins "github.com/apache/incubator-devlake/plugins/jenkins/impl" jira "github.com/apache/incubator-devlake/plugins/jira/impl" org "github.com/apache/incubator-devlake/plugins/org/impl" @@ -74,6 +75,7 @@ func loadGoPlugins() []plugin.PluginMeta { githubGraphql.GithubGraphql{}, gitlab.Gitlab{}, icla.Icla{}, + incidentio.Incidentio{}, jenkins.Jenkins{}, jira.Jira{}, org.Org{}, diff --git a/config-ui/.eslintignore b/config-ui/.eslintignore deleted file mode 100644 index 35be3b14c53..00000000000 --- a/config-ui/.eslintignore +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -dist -node_modules \ No newline at end of file diff --git a/config-ui/.yarn/patches/miller-columns-select-npm-1.4.1-a8bc1f9d49.patch b/config-ui/.yarn/patches/miller-columns-select-npm-1.4.1-a8bc1f9d49.patch new file mode 100644 index 00000000000..c6fa88f303a --- /dev/null +++ b/config-ui/.yarn/patches/miller-columns-select-npm-1.4.1-a8bc1f9d49.patch @@ -0,0 +1,34 @@ +diff --git a/dist/miller-columns-select.es.js b/dist/miller-columns-select.es.js +index d3726ca106906079b3f83d0e854f184ba5d2e086..35f5b37da25429284827c55f0319bc824c38608f 100644 +--- a/dist/miller-columns-select.es.js ++++ b/dist/miller-columns-select.es.js +@@ -1,4 +1,5 @@ + import ze, { useCallback as ie, useMemo as M, useState as Re, useEffect as Be } from "react"; ++import { jsx as __mcsJsx, jsxs as __mcsJsxs, Fragment as __mcsFragment } from "react/jsx-runtime"; + import qe from "classnames"; + import K from "styled-components"; + import gr from "react-infinite-scroll-component"; +@@ -801,9 +802,7 @@ Check the top-level render call using <` + n + ">."); + ae.Fragment = f, ae.jsx = yr, ae.jsxs = mr; + }()), ae; + } +-(function(a) { +- process.env.NODE_ENV === "production" ? a.exports = Rr() : a.exports = Er(); +-})(Ee); ++Ee.exports = { jsx: __mcsJsx, jsxs: __mcsJsxs, Fragment: __mcsFragment }; + const O = Ee.exports.jsx, oe = Ee.exports.jsxs, Je = ({ + status: a, + children: i, +diff --git a/dist/miller-columns-select.umd.js b/dist/miller-columns-select.umd.js +index 33af8fe62838c0f4ba2e7257b33ac0d49221f01f..bd46431f146059556150af36e9417f5a1496bfed 100644 +--- a/dist/miller-columns-select.umd.js ++++ b/dist/miller-columns-select.umd.js +@@ -104,7 +104,7 @@ Check the render method of \``+e+"`."}return""}}function Ir(e){{if(e!==void 0){v + + Check your code at `+r+":"+a+"."}return""}}var ze={};function Pr(e){{var r=Me();if(!r){var a=typeof e=="string"?e:e.displayName||e.name;a&&(r=` + +-Check the top-level render call using <`+a+">.")}return r}}function Be(e,r){{if(!e._store||e._store.validated||e.key!=null)return;e._store.validated=!0;var a=Pr(r);if(ze[a])return;ze[a]=!0;var l="";e&&e._owner&&e._owner!==ke.current&&(l=" It was passed a child from "+N(e._owner.type)+"."),H(e),E('Each child in a list should have a unique "key" prop.%s%s See https://reactjs.org/link/warning-keys for more information.',a,l),H(null)}}function Je(e,r){{if(typeof e!="object")return;if(xe(e))for(var a=0;a",p=" Did you accidentally export a JSX literal instead of a component?"):w=typeof e,E("React.jsx: type is invalid -- expected a string (for built-in components) or a class/function (for composite components) but got: %s.%s",w,p)}var C=Or(e,r,a,m,_);if(C==null)return C;if(v){var j=r.children;if(j!==void 0)if(l)if(xe(j)){for(var ee=0;ee{const f=Te.default("checkbox",{"checkbox-checked":t===S.checked||Array.isArray(t)&&t.includes(S.checked),"checkbox-indeterminate":t===S.indeterminate||Array.isArray(t)&&t.includes(S.indeterminate),"checkbox-disabled":t===S.disabled||Array.isArray(t)&&(t==null?void 0:t.includes(S.disabled))});return ne(er,{onClick:u=>{t!==S.disabled&&(c==null||c(u))},children:[O("span",{className:f}),i&&O("span",{className:"text",children:i})]})},nr=K.default.div` ++Check the top-level render call using <`+a+">.")}return r}}function Be(e,r){{if(!e._store||e._store.validated||e.key!=null)return;e._store.validated=!0;var a=Pr(r);if(ze[a])return;ze[a]=!0;var l="";e&&e._owner&&e._owner!==ke.current&&(l=" It was passed a child from "+N(e._owner.type)+"."),H(e),E('Each child in a list should have a unique "key" prop.%s%s See https://reactjs.org/link/warning-keys for more information.',a,l),H(null)}}function Je(e,r){{if(typeof e!="object")return;if(xe(e))for(var a=0;a",p=" Did you accidentally export a JSX literal instead of a component?"):w=typeof e,E("React.jsx: type is invalid -- expected a string (for built-in components) or a class/function (for composite components) but got: %s.%s",w,p)}var C=Or(e,r,a,m,_);if(C==null)return C;if(v){var j=r.children;if(j!==void 0)if(l)if(xe(j)){for(var ee=0;ee{const f=Te.default("checkbox",{"checkbox-checked":t===S.checked||Array.isArray(t)&&t.includes(S.checked),"checkbox-indeterminate":t===S.indeterminate||Array.isArray(t)&&t.includes(S.indeterminate),"checkbox-disabled":t===S.disabled||Array.isArray(t)&&(t==null?void 0:t.includes(S.disabled))});return ne(er,{onClick:u=>{t!==S.disabled&&(c==null||c(u))},children:[O("span",{className:f}),i&&O("span",{className:"text",children:i})]})},nr=K.default.div` + ${({count:t})=>` + flex: 0 0 ${100/t}%; + width: ${100/t}%; diff --git a/config-ui/.yarn/releases/yarn-3.4.1.cjs b/config-ui/.yarn/releases/yarn-3.4.1.cjs deleted file mode 100755 index c4f99e139c6..00000000000 --- a/config-ui/.yarn/releases/yarn-3.4.1.cjs +++ /dev/null @@ -1,911 +0,0 @@ -#!/usr/bin/env node -/* eslint-disable */ -//prettier-ignore -/*! -The yarn-3.4.1.cjs file is auto-generated by Yarn package manager under: - -BSD 2-Clause License - -For Yarn software - -Copyright (c) 2016-present, Yarn Contributors. All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -/*! - * This file also includes multiple third-party codes, each under its respective license. - - * The licenses for these third-party codes are detailed at the end of this file and the project's LICENSE file. - - * Users and developers are encouraged to refer to the LICENSE file for specific licensing information pertaining to each third-party code segment included herein. -*/ -(()=>{var Mue=Object.create;var Wb=Object.defineProperty;var Kue=Object.getOwnPropertyDescriptor;var Uue=Object.getOwnPropertyNames;var Hue=Object.getPrototypeOf,Gue=Object.prototype.hasOwnProperty;var J=(r=>typeof require<"u"?require:typeof Proxy<"u"?new Proxy(r,{get:(e,t)=>(typeof require<"u"?require:e)[t]}):r)(function(r){if(typeof require<"u")return require.apply(this,arguments);throw new Error('Dynamic require of "'+r+'" is not supported')});var Yue=(r,e)=>()=>(r&&(e=r(r=0)),e);var w=(r,e)=>()=>(e||r((e={exports:{}}).exports,e),e.exports),ut=(r,e)=>{for(var t in e)Wb(r,t,{get:e[t],enumerable:!0})},jue=(r,e,t,i)=>{if(e&&typeof e=="object"||typeof e=="function")for(let n of Uue(e))!Gue.call(r,n)&&n!==t&&Wb(r,n,{get:()=>e[n],enumerable:!(i=Kue(e,n))||i.enumerable});return r};var Pe=(r,e,t)=>(t=r!=null?Mue(Hue(r)):{},jue(e||!r||!r.__esModule?Wb(t,"default",{value:r,enumerable:!0}):t,r));var _1=w((O7e,X1)=>{X1.exports=V1;V1.sync=uge;var W1=J("fs");function cge(r,e){var t=e.pathExt!==void 0?e.pathExt:process.env.PATHEXT;if(!t||(t=t.split(";"),t.indexOf("")!==-1))return!0;for(var i=0;i{tK.exports=$1;$1.sync=gge;var Z1=J("fs");function $1(r,e,t){Z1.stat(r,function(i,n){t(i,i?!1:eK(n,e))})}function gge(r,e){return eK(Z1.statSync(r),e)}function eK(r,e){return r.isFile()&&fge(r,e)}function fge(r,e){var t=r.mode,i=r.uid,n=r.gid,s=e.uid!==void 0?e.uid:process.getuid&&process.getuid(),o=e.gid!==void 0?e.gid:process.getgid&&process.getgid(),a=parseInt("100",8),l=parseInt("010",8),c=parseInt("001",8),u=a|l,g=t&c||t&l&&n===o||t&a&&i===s||t&u&&s===0;return g}});var nK=w((U7e,iK)=>{var K7e=J("fs"),_E;process.platform==="win32"||global.TESTING_WINDOWS?_E=_1():_E=rK();iK.exports=uS;uS.sync=hge;function uS(r,e,t){if(typeof e=="function"&&(t=e,e={}),!t){if(typeof Promise!="function")throw new TypeError("callback not provided");return new Promise(function(i,n){uS(r,e||{},function(s,o){s?n(s):i(o)})})}_E(r,e||{},function(i,n){i&&(i.code==="EACCES"||e&&e.ignoreErrors)&&(i=null,n=!1),t(i,n)})}function hge(r,e){try{return _E.sync(r,e||{})}catch(t){if(e&&e.ignoreErrors||t.code==="EACCES")return!1;throw t}}});var uK=w((H7e,cK)=>{var Ig=process.platform==="win32"||process.env.OSTYPE==="cygwin"||process.env.OSTYPE==="msys",sK=J("path"),pge=Ig?";":":",oK=nK(),aK=r=>Object.assign(new Error(`not found: ${r}`),{code:"ENOENT"}),AK=(r,e)=>{let t=e.colon||pge,i=r.match(/\//)||Ig&&r.match(/\\/)?[""]:[...Ig?[process.cwd()]:[],...(e.path||process.env.PATH||"").split(t)],n=Ig?e.pathExt||process.env.PATHEXT||".EXE;.CMD;.BAT;.COM":"",s=Ig?n.split(t):[""];return Ig&&r.indexOf(".")!==-1&&s[0]!==""&&s.unshift(""),{pathEnv:i,pathExt:s,pathExtExe:n}},lK=(r,e,t)=>{typeof e=="function"&&(t=e,e={}),e||(e={});let{pathEnv:i,pathExt:n,pathExtExe:s}=AK(r,e),o=[],a=c=>new Promise((u,g)=>{if(c===i.length)return e.all&&o.length?u(o):g(aK(r));let f=i[c],h=/^".*"$/.test(f)?f.slice(1,-1):f,p=sK.join(h,r),C=!h&&/^\.[\\\/]/.test(r)?r.slice(0,2)+p:p;u(l(C,c,0))}),l=(c,u,g)=>new Promise((f,h)=>{if(g===n.length)return f(a(u+1));let p=n[g];oK(c+p,{pathExt:s},(C,y)=>{if(!C&&y)if(e.all)o.push(c+p);else return f(c+p);return f(l(c,u,g+1))})});return t?a(0).then(c=>t(null,c),t):a(0)},dge=(r,e)=>{e=e||{};let{pathEnv:t,pathExt:i,pathExtExe:n}=AK(r,e),s=[];for(let o=0;o{"use strict";var gK=(r={})=>{let e=r.env||process.env;return(r.platform||process.platform)!=="win32"?"PATH":Object.keys(e).reverse().find(i=>i.toUpperCase()==="PATH")||"Path"};gS.exports=gK;gS.exports.default=gK});var CK=w((Y7e,dK)=>{"use strict";var hK=J("path"),Cge=uK(),mge=fK();function pK(r,e){let t=r.options.env||process.env,i=process.cwd(),n=r.options.cwd!=null,s=n&&process.chdir!==void 0&&!process.chdir.disabled;if(s)try{process.chdir(r.options.cwd)}catch{}let o;try{o=Cge.sync(r.command,{path:t[mge({env:t})],pathExt:e?hK.delimiter:void 0})}catch{}finally{s&&process.chdir(i)}return o&&(o=hK.resolve(n?r.options.cwd:"",o)),o}function Ege(r){return pK(r)||pK(r,!0)}dK.exports=Ege});var mK=w((j7e,hS)=>{"use strict";var fS=/([()\][%!^"`<>&|;, *?])/g;function Ige(r){return r=r.replace(fS,"^$1"),r}function yge(r,e){return r=`${r}`,r=r.replace(/(\\*)"/g,'$1$1\\"'),r=r.replace(/(\\*)$/,"$1$1"),r=`"${r}"`,r=r.replace(fS,"^$1"),e&&(r=r.replace(fS,"^$1")),r}hS.exports.command=Ige;hS.exports.argument=yge});var IK=w((q7e,EK)=>{"use strict";EK.exports=/^#!(.*)/});var wK=w((J7e,yK)=>{"use strict";var wge=IK();yK.exports=(r="")=>{let e=r.match(wge);if(!e)return null;let[t,i]=e[0].replace(/#! ?/,"").split(" "),n=t.split("/").pop();return n==="env"?i:i?`${n} ${i}`:n}});var QK=w((W7e,BK)=>{"use strict";var pS=J("fs"),Bge=wK();function Qge(r){let t=Buffer.alloc(150),i;try{i=pS.openSync(r,"r"),pS.readSync(i,t,0,150,0),pS.closeSync(i)}catch{}return Bge(t.toString())}BK.exports=Qge});var xK=w((z7e,vK)=>{"use strict";var bge=J("path"),bK=CK(),SK=mK(),Sge=QK(),vge=process.platform==="win32",xge=/\.(?:com|exe)$/i,Pge=/node_modules[\\/].bin[\\/][^\\/]+\.cmd$/i;function Dge(r){r.file=bK(r);let e=r.file&&Sge(r.file);return e?(r.args.unshift(r.file),r.command=e,bK(r)):r.file}function kge(r){if(!vge)return r;let e=Dge(r),t=!xge.test(e);if(r.options.forceShell||t){let i=Pge.test(e);r.command=bge.normalize(r.command),r.command=SK.command(r.command),r.args=r.args.map(s=>SK.argument(s,i));let n=[r.command].concat(r.args).join(" ");r.args=["/d","/s","/c",`"${n}"`],r.command=process.env.comspec||"cmd.exe",r.options.windowsVerbatimArguments=!0}return r}function Rge(r,e,t){e&&!Array.isArray(e)&&(t=e,e=null),e=e?e.slice(0):[],t=Object.assign({},t);let i={command:r,args:e,options:t,file:void 0,original:{command:r,args:e}};return t.shell?i:kge(i)}vK.exports=Rge});var kK=w((V7e,DK)=>{"use strict";var dS=process.platform==="win32";function CS(r,e){return Object.assign(new Error(`${e} ${r.command} ENOENT`),{code:"ENOENT",errno:"ENOENT",syscall:`${e} ${r.command}`,path:r.command,spawnargs:r.args})}function Fge(r,e){if(!dS)return;let t=r.emit;r.emit=function(i,n){if(i==="exit"){let s=PK(n,e,"spawn");if(s)return t.call(r,"error",s)}return t.apply(r,arguments)}}function PK(r,e){return dS&&r===1&&!e.file?CS(e.original,"spawn"):null}function Nge(r,e){return dS&&r===1&&!e.file?CS(e.original,"spawnSync"):null}DK.exports={hookChildProcess:Fge,verifyENOENT:PK,verifyENOENTSync:Nge,notFoundError:CS}});var IS=w((X7e,yg)=>{"use strict";var RK=J("child_process"),mS=xK(),ES=kK();function FK(r,e,t){let i=mS(r,e,t),n=RK.spawn(i.command,i.args,i.options);return ES.hookChildProcess(n,i),n}function Lge(r,e,t){let i=mS(r,e,t),n=RK.spawnSync(i.command,i.args,i.options);return n.error=n.error||ES.verifyENOENTSync(n.status,i),n}yg.exports=FK;yg.exports.spawn=FK;yg.exports.sync=Lge;yg.exports._parse=mS;yg.exports._enoent=ES});var LK=w((_7e,NK)=>{"use strict";function Tge(r,e){function t(){this.constructor=r}t.prototype=e.prototype,r.prototype=new t}function Ml(r,e,t,i){this.message=r,this.expected=e,this.found=t,this.location=i,this.name="SyntaxError",typeof Error.captureStackTrace=="function"&&Error.captureStackTrace(this,Ml)}Tge(Ml,Error);Ml.buildMessage=function(r,e){var t={literal:function(c){return'"'+n(c.text)+'"'},class:function(c){var u="",g;for(g=0;g0){for(g=1,f=1;g>",ie=me(">>",!1),de=">&",tt=me(">&",!1),Pt=">",It=me(">",!1),Or="<<<",ii=me("<<<",!1),gi="<&",hr=me("<&",!1),fi="<",ni=me("<",!1),Ls=function(m){return{type:"argument",segments:[].concat(...m)}},pr=function(m){return m},Ei="$'",_n=me("$'",!1),oa="'",aA=me("'",!1),eg=function(m){return[{type:"text",text:m}]},Zn='""',AA=me('""',!1),aa=function(){return{type:"text",text:""}},up='"',lA=me('"',!1),cA=function(m){return m},wr=function(m){return{type:"arithmetic",arithmetic:m,quoted:!0}},wl=function(m){return{type:"shell",shell:m,quoted:!0}},tg=function(m){return{type:"variable",...m,quoted:!0}},po=function(m){return{type:"text",text:m}},rg=function(m){return{type:"arithmetic",arithmetic:m,quoted:!1}},gp=function(m){return{type:"shell",shell:m,quoted:!1}},fp=function(m){return{type:"variable",...m,quoted:!1}},vr=function(m){return{type:"glob",pattern:m}},se=/^[^']/,Co=Je(["'"],!0,!1),Dn=function(m){return m.join("")},ig=/^[^$"]/,Qt=Je(["$",'"'],!0,!1),Bl=`\\ -`,kn=me(`\\ -`,!1),$n=function(){return""},es="\\",gt=me("\\",!1),mo=/^[\\$"`]/,At=Je(["\\","$",'"',"`"],!1,!1),an=function(m){return m},S="\\a",Tt=me("\\a",!1),ng=function(){return"a"},Ql="\\b",hp=me("\\b",!1),pp=function(){return"\b"},dp=/^[Ee]/,Cp=Je(["E","e"],!1,!1),mp=function(){return"\x1B"},G="\\f",yt=me("\\f",!1),uA=function(){return"\f"},ji="\\n",bl=me("\\n",!1),Xe=function(){return` -`},Aa="\\r",sg=me("\\r",!1),bE=function(){return"\r"},Ep="\\t",SE=me("\\t",!1),ar=function(){return" "},Rn="\\v",Sl=me("\\v",!1),Ip=function(){return"\v"},Ts=/^[\\'"?]/,la=Je(["\\","'",'"',"?"],!1,!1),An=function(m){return String.fromCharCode(parseInt(m,16))},Te="\\x",og=me("\\x",!1),vl="\\u",Os=me("\\u",!1),xl="\\U",gA=me("\\U",!1),ag=function(m){return String.fromCodePoint(parseInt(m,16))},Ag=/^[0-7]/,ca=Je([["0","7"]],!1,!1),ua=/^[0-9a-fA-f]/,rt=Je([["0","9"],["a","f"],["A","f"]],!1,!1),Eo=nt(),fA="-",Pl=me("-",!1),Ms="+",Dl=me("+",!1),vE=".",yp=me(".",!1),lg=function(m,b,N){return{type:"number",value:(m==="-"?-1:1)*parseFloat(b.join("")+"."+N.join(""))}},wp=function(m,b){return{type:"number",value:(m==="-"?-1:1)*parseInt(b.join(""))}},xE=function(m){return{type:"variable",...m}},kl=function(m){return{type:"variable",name:m}},PE=function(m){return m},cg="*",hA=me("*",!1),Rr="/",DE=me("/",!1),Ks=function(m,b,N){return{type:b==="*"?"multiplication":"division",right:N}},Us=function(m,b){return b.reduce((N,U)=>({left:N,...U}),m)},ug=function(m,b,N){return{type:b==="+"?"addition":"subtraction",right:N}},pA="$((",R=me("$((",!1),q="))",Ce=me("))",!1),Ke=function(m){return m},Re="$(",ze=me("$(",!1),dt=function(m){return m},Ft="${",Fn=me("${",!1),Db=":-",$M=me(":-",!1),e1=function(m,b){return{name:m,defaultValue:b}},kb=":-}",t1=me(":-}",!1),r1=function(m){return{name:m,defaultValue:[]}},Rb=":+",i1=me(":+",!1),n1=function(m,b){return{name:m,alternativeValue:b}},Fb=":+}",s1=me(":+}",!1),o1=function(m){return{name:m,alternativeValue:[]}},Nb=function(m){return{name:m}},a1="$",A1=me("$",!1),l1=function(m){return e.isGlobPattern(m)},c1=function(m){return m},Lb=/^[a-zA-Z0-9_]/,Tb=Je([["a","z"],["A","Z"],["0","9"],"_"],!1,!1),Ob=function(){return T()},Mb=/^[$@*?#a-zA-Z0-9_\-]/,Kb=Je(["$","@","*","?","#",["a","z"],["A","Z"],["0","9"],"_","-"],!1,!1),u1=/^[(){}<>$|&; \t"']/,gg=Je(["(",")","{","}","<",">","$","|","&",";"," "," ",'"',"'"],!1,!1),Ub=/^[<>&; \t"']/,Hb=Je(["<",">","&",";"," "," ",'"',"'"],!1,!1),kE=/^[ \t]/,RE=Je([" "," "],!1,!1),Q=0,Me=0,dA=[{line:1,column:1}],d=0,E=[],I=0,k;if("startRule"in e){if(!(e.startRule in i))throw new Error(`Can't start parsing from rule "`+e.startRule+'".');n=i[e.startRule]}function T(){return r.substring(Me,Q)}function _(){return Et(Me,Q)}function te(m,b){throw b=b!==void 0?b:Et(Me,Q),ki([lt(m)],r.substring(Me,Q),b)}function Be(m,b){throw b=b!==void 0?b:Et(Me,Q),Nn(m,b)}function me(m,b){return{type:"literal",text:m,ignoreCase:b}}function Je(m,b,N){return{type:"class",parts:m,inverted:b,ignoreCase:N}}function nt(){return{type:"any"}}function wt(){return{type:"end"}}function lt(m){return{type:"other",description:m}}function it(m){var b=dA[m],N;if(b)return b;for(N=m-1;!dA[N];)N--;for(b=dA[N],b={line:b.line,column:b.column};Nd&&(d=Q,E=[]),E.push(m))}function Nn(m,b){return new Ml(m,null,null,b)}function ki(m,b,N){return new Ml(Ml.buildMessage(m,b),m,b,N)}function CA(){var m,b;return m=Q,b=Mr(),b===t&&(b=null),b!==t&&(Me=m,b=s(b)),m=b,m}function Mr(){var m,b,N,U,ce;if(m=Q,b=Kr(),b!==t){for(N=[],U=He();U!==t;)N.push(U),U=He();N!==t?(U=ga(),U!==t?(ce=ts(),ce===t&&(ce=null),ce!==t?(Me=m,b=o(b,U,ce),m=b):(Q=m,m=t)):(Q=m,m=t)):(Q=m,m=t)}else Q=m,m=t;if(m===t)if(m=Q,b=Kr(),b!==t){for(N=[],U=He();U!==t;)N.push(U),U=He();N!==t?(U=ga(),U===t&&(U=null),U!==t?(Me=m,b=a(b,U),m=b):(Q=m,m=t)):(Q=m,m=t)}else Q=m,m=t;return m}function ts(){var m,b,N,U,ce;for(m=Q,b=[],N=He();N!==t;)b.push(N),N=He();if(b!==t)if(N=Mr(),N!==t){for(U=[],ce=He();ce!==t;)U.push(ce),ce=He();U!==t?(Me=m,b=l(N),m=b):(Q=m,m=t)}else Q=m,m=t;else Q=m,m=t;return m}function ga(){var m;return r.charCodeAt(Q)===59?(m=c,Q++):(m=t,I===0&&Qe(u)),m===t&&(r.charCodeAt(Q)===38?(m=g,Q++):(m=t,I===0&&Qe(f))),m}function Kr(){var m,b,N;return m=Q,b=g1(),b!==t?(N=yue(),N===t&&(N=null),N!==t?(Me=m,b=h(b,N),m=b):(Q=m,m=t)):(Q=m,m=t),m}function yue(){var m,b,N,U,ce,Se,ht;for(m=Q,b=[],N=He();N!==t;)b.push(N),N=He();if(b!==t)if(N=wue(),N!==t){for(U=[],ce=He();ce!==t;)U.push(ce),ce=He();if(U!==t)if(ce=Kr(),ce!==t){for(Se=[],ht=He();ht!==t;)Se.push(ht),ht=He();Se!==t?(Me=m,b=p(N,ce),m=b):(Q=m,m=t)}else Q=m,m=t;else Q=m,m=t}else Q=m,m=t;else Q=m,m=t;return m}function wue(){var m;return r.substr(Q,2)===C?(m=C,Q+=2):(m=t,I===0&&Qe(y)),m===t&&(r.substr(Q,2)===B?(m=B,Q+=2):(m=t,I===0&&Qe(v))),m}function g1(){var m,b,N;return m=Q,b=bue(),b!==t?(N=Bue(),N===t&&(N=null),N!==t?(Me=m,b=D(b,N),m=b):(Q=m,m=t)):(Q=m,m=t),m}function Bue(){var m,b,N,U,ce,Se,ht;for(m=Q,b=[],N=He();N!==t;)b.push(N),N=He();if(b!==t)if(N=Que(),N!==t){for(U=[],ce=He();ce!==t;)U.push(ce),ce=He();if(U!==t)if(ce=g1(),ce!==t){for(Se=[],ht=He();ht!==t;)Se.push(ht),ht=He();Se!==t?(Me=m,b=L(N,ce),m=b):(Q=m,m=t)}else Q=m,m=t;else Q=m,m=t}else Q=m,m=t;else Q=m,m=t;return m}function Que(){var m;return r.substr(Q,2)===H?(m=H,Q+=2):(m=t,I===0&&Qe(j)),m===t&&(r.charCodeAt(Q)===124?(m=$,Q++):(m=t,I===0&&Qe(V))),m}function FE(){var m,b,N,U,ce,Se;if(m=Q,b=Q1(),b!==t)if(r.charCodeAt(Q)===61?(N=W,Q++):(N=t,I===0&&Qe(Z)),N!==t)if(U=p1(),U!==t){for(ce=[],Se=He();Se!==t;)ce.push(Se),Se=He();ce!==t?(Me=m,b=A(b,U),m=b):(Q=m,m=t)}else Q=m,m=t;else Q=m,m=t;else Q=m,m=t;if(m===t)if(m=Q,b=Q1(),b!==t)if(r.charCodeAt(Q)===61?(N=W,Q++):(N=t,I===0&&Qe(Z)),N!==t){for(U=[],ce=He();ce!==t;)U.push(ce),ce=He();U!==t?(Me=m,b=ae(b),m=b):(Q=m,m=t)}else Q=m,m=t;else Q=m,m=t;return m}function bue(){var m,b,N,U,ce,Se,ht,Bt,Jr,hi,rs;for(m=Q,b=[],N=He();N!==t;)b.push(N),N=He();if(b!==t)if(r.charCodeAt(Q)===40?(N=ge,Q++):(N=t,I===0&&Qe(re)),N!==t){for(U=[],ce=He();ce!==t;)U.push(ce),ce=He();if(U!==t)if(ce=Mr(),ce!==t){for(Se=[],ht=He();ht!==t;)Se.push(ht),ht=He();if(Se!==t)if(r.charCodeAt(Q)===41?(ht=O,Q++):(ht=t,I===0&&Qe(F)),ht!==t){for(Bt=[],Jr=He();Jr!==t;)Bt.push(Jr),Jr=He();if(Bt!==t){for(Jr=[],hi=Bp();hi!==t;)Jr.push(hi),hi=Bp();if(Jr!==t){for(hi=[],rs=He();rs!==t;)hi.push(rs),rs=He();hi!==t?(Me=m,b=ue(ce,Jr),m=b):(Q=m,m=t)}else Q=m,m=t}else Q=m,m=t}else Q=m,m=t;else Q=m,m=t}else Q=m,m=t;else Q=m,m=t}else Q=m,m=t;else Q=m,m=t;if(m===t){for(m=Q,b=[],N=He();N!==t;)b.push(N),N=He();if(b!==t)if(r.charCodeAt(Q)===123?(N=he,Q++):(N=t,I===0&&Qe(ke)),N!==t){for(U=[],ce=He();ce!==t;)U.push(ce),ce=He();if(U!==t)if(ce=Mr(),ce!==t){for(Se=[],ht=He();ht!==t;)Se.push(ht),ht=He();if(Se!==t)if(r.charCodeAt(Q)===125?(ht=Fe,Q++):(ht=t,I===0&&Qe(Ne)),ht!==t){for(Bt=[],Jr=He();Jr!==t;)Bt.push(Jr),Jr=He();if(Bt!==t){for(Jr=[],hi=Bp();hi!==t;)Jr.push(hi),hi=Bp();if(Jr!==t){for(hi=[],rs=He();rs!==t;)hi.push(rs),rs=He();hi!==t?(Me=m,b=oe(ce,Jr),m=b):(Q=m,m=t)}else Q=m,m=t}else Q=m,m=t}else Q=m,m=t;else Q=m,m=t}else Q=m,m=t;else Q=m,m=t}else Q=m,m=t;else Q=m,m=t;if(m===t){for(m=Q,b=[],N=He();N!==t;)b.push(N),N=He();if(b!==t){for(N=[],U=FE();U!==t;)N.push(U),U=FE();if(N!==t){for(U=[],ce=He();ce!==t;)U.push(ce),ce=He();if(U!==t){if(ce=[],Se=h1(),Se!==t)for(;Se!==t;)ce.push(Se),Se=h1();else ce=t;if(ce!==t){for(Se=[],ht=He();ht!==t;)Se.push(ht),ht=He();Se!==t?(Me=m,b=le(N,ce),m=b):(Q=m,m=t)}else Q=m,m=t}else Q=m,m=t}else Q=m,m=t}else Q=m,m=t;if(m===t){for(m=Q,b=[],N=He();N!==t;)b.push(N),N=He();if(b!==t){if(N=[],U=FE(),U!==t)for(;U!==t;)N.push(U),U=FE();else N=t;if(N!==t){for(U=[],ce=He();ce!==t;)U.push(ce),ce=He();U!==t?(Me=m,b=we(N),m=b):(Q=m,m=t)}else Q=m,m=t}else Q=m,m=t}}}return m}function f1(){var m,b,N,U,ce;for(m=Q,b=[],N=He();N!==t;)b.push(N),N=He();if(b!==t){if(N=[],U=NE(),U!==t)for(;U!==t;)N.push(U),U=NE();else N=t;if(N!==t){for(U=[],ce=He();ce!==t;)U.push(ce),ce=He();U!==t?(Me=m,b=fe(N),m=b):(Q=m,m=t)}else Q=m,m=t}else Q=m,m=t;return m}function h1(){var m,b,N;for(m=Q,b=[],N=He();N!==t;)b.push(N),N=He();if(b!==t?(N=Bp(),N!==t?(Me=m,b=Ae(N),m=b):(Q=m,m=t)):(Q=m,m=t),m===t){for(m=Q,b=[],N=He();N!==t;)b.push(N),N=He();b!==t?(N=NE(),N!==t?(Me=m,b=Ae(N),m=b):(Q=m,m=t)):(Q=m,m=t)}return m}function Bp(){var m,b,N,U,ce;for(m=Q,b=[],N=He();N!==t;)b.push(N),N=He();return b!==t?(qe.test(r.charAt(Q))?(N=r.charAt(Q),Q++):(N=t,I===0&&Qe(ne)),N===t&&(N=null),N!==t?(U=Sue(),U!==t?(ce=NE(),ce!==t?(Me=m,b=Y(N,U,ce),m=b):(Q=m,m=t)):(Q=m,m=t)):(Q=m,m=t)):(Q=m,m=t),m}function Sue(){var m;return r.substr(Q,2)===pe?(m=pe,Q+=2):(m=t,I===0&&Qe(ie)),m===t&&(r.substr(Q,2)===de?(m=de,Q+=2):(m=t,I===0&&Qe(tt)),m===t&&(r.charCodeAt(Q)===62?(m=Pt,Q++):(m=t,I===0&&Qe(It)),m===t&&(r.substr(Q,3)===Or?(m=Or,Q+=3):(m=t,I===0&&Qe(ii)),m===t&&(r.substr(Q,2)===gi?(m=gi,Q+=2):(m=t,I===0&&Qe(hr)),m===t&&(r.charCodeAt(Q)===60?(m=fi,Q++):(m=t,I===0&&Qe(ni))))))),m}function NE(){var m,b,N;for(m=Q,b=[],N=He();N!==t;)b.push(N),N=He();return b!==t?(N=p1(),N!==t?(Me=m,b=Ae(N),m=b):(Q=m,m=t)):(Q=m,m=t),m}function p1(){var m,b,N;if(m=Q,b=[],N=d1(),N!==t)for(;N!==t;)b.push(N),N=d1();else b=t;return b!==t&&(Me=m,b=Ls(b)),m=b,m}function d1(){var m,b;return m=Q,b=vue(),b!==t&&(Me=m,b=pr(b)),m=b,m===t&&(m=Q,b=xue(),b!==t&&(Me=m,b=pr(b)),m=b,m===t&&(m=Q,b=Pue(),b!==t&&(Me=m,b=pr(b)),m=b,m===t&&(m=Q,b=Due(),b!==t&&(Me=m,b=pr(b)),m=b))),m}function vue(){var m,b,N,U;return m=Q,r.substr(Q,2)===Ei?(b=Ei,Q+=2):(b=t,I===0&&Qe(_n)),b!==t?(N=Fue(),N!==t?(r.charCodeAt(Q)===39?(U=oa,Q++):(U=t,I===0&&Qe(aA)),U!==t?(Me=m,b=eg(N),m=b):(Q=m,m=t)):(Q=m,m=t)):(Q=m,m=t),m}function xue(){var m,b,N,U;return m=Q,r.charCodeAt(Q)===39?(b=oa,Q++):(b=t,I===0&&Qe(aA)),b!==t?(N=kue(),N!==t?(r.charCodeAt(Q)===39?(U=oa,Q++):(U=t,I===0&&Qe(aA)),U!==t?(Me=m,b=eg(N),m=b):(Q=m,m=t)):(Q=m,m=t)):(Q=m,m=t),m}function Pue(){var m,b,N,U;if(m=Q,r.substr(Q,2)===Zn?(b=Zn,Q+=2):(b=t,I===0&&Qe(AA)),b!==t&&(Me=m,b=aa()),m=b,m===t)if(m=Q,r.charCodeAt(Q)===34?(b=up,Q++):(b=t,I===0&&Qe(lA)),b!==t){for(N=[],U=C1();U!==t;)N.push(U),U=C1();N!==t?(r.charCodeAt(Q)===34?(U=up,Q++):(U=t,I===0&&Qe(lA)),U!==t?(Me=m,b=cA(N),m=b):(Q=m,m=t)):(Q=m,m=t)}else Q=m,m=t;return m}function Due(){var m,b,N;if(m=Q,b=[],N=m1(),N!==t)for(;N!==t;)b.push(N),N=m1();else b=t;return b!==t&&(Me=m,b=cA(b)),m=b,m}function C1(){var m,b;return m=Q,b=w1(),b!==t&&(Me=m,b=wr(b)),m=b,m===t&&(m=Q,b=B1(),b!==t&&(Me=m,b=wl(b)),m=b,m===t&&(m=Q,b=qb(),b!==t&&(Me=m,b=tg(b)),m=b,m===t&&(m=Q,b=Rue(),b!==t&&(Me=m,b=po(b)),m=b))),m}function m1(){var m,b;return m=Q,b=w1(),b!==t&&(Me=m,b=rg(b)),m=b,m===t&&(m=Q,b=B1(),b!==t&&(Me=m,b=gp(b)),m=b,m===t&&(m=Q,b=qb(),b!==t&&(Me=m,b=fp(b)),m=b,m===t&&(m=Q,b=Tue(),b!==t&&(Me=m,b=vr(b)),m=b,m===t&&(m=Q,b=Lue(),b!==t&&(Me=m,b=po(b)),m=b)))),m}function kue(){var m,b,N;for(m=Q,b=[],se.test(r.charAt(Q))?(N=r.charAt(Q),Q++):(N=t,I===0&&Qe(Co));N!==t;)b.push(N),se.test(r.charAt(Q))?(N=r.charAt(Q),Q++):(N=t,I===0&&Qe(Co));return b!==t&&(Me=m,b=Dn(b)),m=b,m}function Rue(){var m,b,N;if(m=Q,b=[],N=E1(),N===t&&(ig.test(r.charAt(Q))?(N=r.charAt(Q),Q++):(N=t,I===0&&Qe(Qt))),N!==t)for(;N!==t;)b.push(N),N=E1(),N===t&&(ig.test(r.charAt(Q))?(N=r.charAt(Q),Q++):(N=t,I===0&&Qe(Qt)));else b=t;return b!==t&&(Me=m,b=Dn(b)),m=b,m}function E1(){var m,b,N;return m=Q,r.substr(Q,2)===Bl?(b=Bl,Q+=2):(b=t,I===0&&Qe(kn)),b!==t&&(Me=m,b=$n()),m=b,m===t&&(m=Q,r.charCodeAt(Q)===92?(b=es,Q++):(b=t,I===0&&Qe(gt)),b!==t?(mo.test(r.charAt(Q))?(N=r.charAt(Q),Q++):(N=t,I===0&&Qe(At)),N!==t?(Me=m,b=an(N),m=b):(Q=m,m=t)):(Q=m,m=t)),m}function Fue(){var m,b,N;for(m=Q,b=[],N=I1(),N===t&&(se.test(r.charAt(Q))?(N=r.charAt(Q),Q++):(N=t,I===0&&Qe(Co)));N!==t;)b.push(N),N=I1(),N===t&&(se.test(r.charAt(Q))?(N=r.charAt(Q),Q++):(N=t,I===0&&Qe(Co)));return b!==t&&(Me=m,b=Dn(b)),m=b,m}function I1(){var m,b,N;return m=Q,r.substr(Q,2)===S?(b=S,Q+=2):(b=t,I===0&&Qe(Tt)),b!==t&&(Me=m,b=ng()),m=b,m===t&&(m=Q,r.substr(Q,2)===Ql?(b=Ql,Q+=2):(b=t,I===0&&Qe(hp)),b!==t&&(Me=m,b=pp()),m=b,m===t&&(m=Q,r.charCodeAt(Q)===92?(b=es,Q++):(b=t,I===0&&Qe(gt)),b!==t?(dp.test(r.charAt(Q))?(N=r.charAt(Q),Q++):(N=t,I===0&&Qe(Cp)),N!==t?(Me=m,b=mp(),m=b):(Q=m,m=t)):(Q=m,m=t),m===t&&(m=Q,r.substr(Q,2)===G?(b=G,Q+=2):(b=t,I===0&&Qe(yt)),b!==t&&(Me=m,b=uA()),m=b,m===t&&(m=Q,r.substr(Q,2)===ji?(b=ji,Q+=2):(b=t,I===0&&Qe(bl)),b!==t&&(Me=m,b=Xe()),m=b,m===t&&(m=Q,r.substr(Q,2)===Aa?(b=Aa,Q+=2):(b=t,I===0&&Qe(sg)),b!==t&&(Me=m,b=bE()),m=b,m===t&&(m=Q,r.substr(Q,2)===Ep?(b=Ep,Q+=2):(b=t,I===0&&Qe(SE)),b!==t&&(Me=m,b=ar()),m=b,m===t&&(m=Q,r.substr(Q,2)===Rn?(b=Rn,Q+=2):(b=t,I===0&&Qe(Sl)),b!==t&&(Me=m,b=Ip()),m=b,m===t&&(m=Q,r.charCodeAt(Q)===92?(b=es,Q++):(b=t,I===0&&Qe(gt)),b!==t?(Ts.test(r.charAt(Q))?(N=r.charAt(Q),Q++):(N=t,I===0&&Qe(la)),N!==t?(Me=m,b=an(N),m=b):(Q=m,m=t)):(Q=m,m=t),m===t&&(m=Nue()))))))))),m}function Nue(){var m,b,N,U,ce,Se,ht,Bt,Jr,hi,rs,Jb;return m=Q,r.charCodeAt(Q)===92?(b=es,Q++):(b=t,I===0&&Qe(gt)),b!==t?(N=Gb(),N!==t?(Me=m,b=An(N),m=b):(Q=m,m=t)):(Q=m,m=t),m===t&&(m=Q,r.substr(Q,2)===Te?(b=Te,Q+=2):(b=t,I===0&&Qe(og)),b!==t?(N=Q,U=Q,ce=Gb(),ce!==t?(Se=Ln(),Se!==t?(ce=[ce,Se],U=ce):(Q=U,U=t)):(Q=U,U=t),U===t&&(U=Gb()),U!==t?N=r.substring(N,Q):N=U,N!==t?(Me=m,b=An(N),m=b):(Q=m,m=t)):(Q=m,m=t),m===t&&(m=Q,r.substr(Q,2)===vl?(b=vl,Q+=2):(b=t,I===0&&Qe(Os)),b!==t?(N=Q,U=Q,ce=Ln(),ce!==t?(Se=Ln(),Se!==t?(ht=Ln(),ht!==t?(Bt=Ln(),Bt!==t?(ce=[ce,Se,ht,Bt],U=ce):(Q=U,U=t)):(Q=U,U=t)):(Q=U,U=t)):(Q=U,U=t),U!==t?N=r.substring(N,Q):N=U,N!==t?(Me=m,b=An(N),m=b):(Q=m,m=t)):(Q=m,m=t),m===t&&(m=Q,r.substr(Q,2)===xl?(b=xl,Q+=2):(b=t,I===0&&Qe(gA)),b!==t?(N=Q,U=Q,ce=Ln(),ce!==t?(Se=Ln(),Se!==t?(ht=Ln(),ht!==t?(Bt=Ln(),Bt!==t?(Jr=Ln(),Jr!==t?(hi=Ln(),hi!==t?(rs=Ln(),rs!==t?(Jb=Ln(),Jb!==t?(ce=[ce,Se,ht,Bt,Jr,hi,rs,Jb],U=ce):(Q=U,U=t)):(Q=U,U=t)):(Q=U,U=t)):(Q=U,U=t)):(Q=U,U=t)):(Q=U,U=t)):(Q=U,U=t)):(Q=U,U=t),U!==t?N=r.substring(N,Q):N=U,N!==t?(Me=m,b=ag(N),m=b):(Q=m,m=t)):(Q=m,m=t)))),m}function Gb(){var m;return Ag.test(r.charAt(Q))?(m=r.charAt(Q),Q++):(m=t,I===0&&Qe(ca)),m}function Ln(){var m;return ua.test(r.charAt(Q))?(m=r.charAt(Q),Q++):(m=t,I===0&&Qe(rt)),m}function Lue(){var m,b,N,U,ce;if(m=Q,b=[],N=Q,r.charCodeAt(Q)===92?(U=es,Q++):(U=t,I===0&&Qe(gt)),U!==t?(r.length>Q?(ce=r.charAt(Q),Q++):(ce=t,I===0&&Qe(Eo)),ce!==t?(Me=N,U=an(ce),N=U):(Q=N,N=t)):(Q=N,N=t),N===t&&(N=Q,U=Q,I++,ce=b1(),I--,ce===t?U=void 0:(Q=U,U=t),U!==t?(r.length>Q?(ce=r.charAt(Q),Q++):(ce=t,I===0&&Qe(Eo)),ce!==t?(Me=N,U=an(ce),N=U):(Q=N,N=t)):(Q=N,N=t)),N!==t)for(;N!==t;)b.push(N),N=Q,r.charCodeAt(Q)===92?(U=es,Q++):(U=t,I===0&&Qe(gt)),U!==t?(r.length>Q?(ce=r.charAt(Q),Q++):(ce=t,I===0&&Qe(Eo)),ce!==t?(Me=N,U=an(ce),N=U):(Q=N,N=t)):(Q=N,N=t),N===t&&(N=Q,U=Q,I++,ce=b1(),I--,ce===t?U=void 0:(Q=U,U=t),U!==t?(r.length>Q?(ce=r.charAt(Q),Q++):(ce=t,I===0&&Qe(Eo)),ce!==t?(Me=N,U=an(ce),N=U):(Q=N,N=t)):(Q=N,N=t));else b=t;return b!==t&&(Me=m,b=Dn(b)),m=b,m}function Yb(){var m,b,N,U,ce,Se;if(m=Q,r.charCodeAt(Q)===45?(b=fA,Q++):(b=t,I===0&&Qe(Pl)),b===t&&(r.charCodeAt(Q)===43?(b=Ms,Q++):(b=t,I===0&&Qe(Dl))),b===t&&(b=null),b!==t){if(N=[],qe.test(r.charAt(Q))?(U=r.charAt(Q),Q++):(U=t,I===0&&Qe(ne)),U!==t)for(;U!==t;)N.push(U),qe.test(r.charAt(Q))?(U=r.charAt(Q),Q++):(U=t,I===0&&Qe(ne));else N=t;if(N!==t)if(r.charCodeAt(Q)===46?(U=vE,Q++):(U=t,I===0&&Qe(yp)),U!==t){if(ce=[],qe.test(r.charAt(Q))?(Se=r.charAt(Q),Q++):(Se=t,I===0&&Qe(ne)),Se!==t)for(;Se!==t;)ce.push(Se),qe.test(r.charAt(Q))?(Se=r.charAt(Q),Q++):(Se=t,I===0&&Qe(ne));else ce=t;ce!==t?(Me=m,b=lg(b,N,ce),m=b):(Q=m,m=t)}else Q=m,m=t;else Q=m,m=t}else Q=m,m=t;if(m===t){if(m=Q,r.charCodeAt(Q)===45?(b=fA,Q++):(b=t,I===0&&Qe(Pl)),b===t&&(r.charCodeAt(Q)===43?(b=Ms,Q++):(b=t,I===0&&Qe(Dl))),b===t&&(b=null),b!==t){if(N=[],qe.test(r.charAt(Q))?(U=r.charAt(Q),Q++):(U=t,I===0&&Qe(ne)),U!==t)for(;U!==t;)N.push(U),qe.test(r.charAt(Q))?(U=r.charAt(Q),Q++):(U=t,I===0&&Qe(ne));else N=t;N!==t?(Me=m,b=wp(b,N),m=b):(Q=m,m=t)}else Q=m,m=t;if(m===t&&(m=Q,b=qb(),b!==t&&(Me=m,b=xE(b)),m=b,m===t&&(m=Q,b=Rl(),b!==t&&(Me=m,b=kl(b)),m=b,m===t)))if(m=Q,r.charCodeAt(Q)===40?(b=ge,Q++):(b=t,I===0&&Qe(re)),b!==t){for(N=[],U=He();U!==t;)N.push(U),U=He();if(N!==t)if(U=y1(),U!==t){for(ce=[],Se=He();Se!==t;)ce.push(Se),Se=He();ce!==t?(r.charCodeAt(Q)===41?(Se=O,Q++):(Se=t,I===0&&Qe(F)),Se!==t?(Me=m,b=PE(U),m=b):(Q=m,m=t)):(Q=m,m=t)}else Q=m,m=t;else Q=m,m=t}else Q=m,m=t}return m}function jb(){var m,b,N,U,ce,Se,ht,Bt;if(m=Q,b=Yb(),b!==t){for(N=[],U=Q,ce=[],Se=He();Se!==t;)ce.push(Se),Se=He();if(ce!==t)if(r.charCodeAt(Q)===42?(Se=cg,Q++):(Se=t,I===0&&Qe(hA)),Se===t&&(r.charCodeAt(Q)===47?(Se=Rr,Q++):(Se=t,I===0&&Qe(DE))),Se!==t){for(ht=[],Bt=He();Bt!==t;)ht.push(Bt),Bt=He();ht!==t?(Bt=Yb(),Bt!==t?(Me=U,ce=Ks(b,Se,Bt),U=ce):(Q=U,U=t)):(Q=U,U=t)}else Q=U,U=t;else Q=U,U=t;for(;U!==t;){for(N.push(U),U=Q,ce=[],Se=He();Se!==t;)ce.push(Se),Se=He();if(ce!==t)if(r.charCodeAt(Q)===42?(Se=cg,Q++):(Se=t,I===0&&Qe(hA)),Se===t&&(r.charCodeAt(Q)===47?(Se=Rr,Q++):(Se=t,I===0&&Qe(DE))),Se!==t){for(ht=[],Bt=He();Bt!==t;)ht.push(Bt),Bt=He();ht!==t?(Bt=Yb(),Bt!==t?(Me=U,ce=Ks(b,Se,Bt),U=ce):(Q=U,U=t)):(Q=U,U=t)}else Q=U,U=t;else Q=U,U=t}N!==t?(Me=m,b=Us(b,N),m=b):(Q=m,m=t)}else Q=m,m=t;return m}function y1(){var m,b,N,U,ce,Se,ht,Bt;if(m=Q,b=jb(),b!==t){for(N=[],U=Q,ce=[],Se=He();Se!==t;)ce.push(Se),Se=He();if(ce!==t)if(r.charCodeAt(Q)===43?(Se=Ms,Q++):(Se=t,I===0&&Qe(Dl)),Se===t&&(r.charCodeAt(Q)===45?(Se=fA,Q++):(Se=t,I===0&&Qe(Pl))),Se!==t){for(ht=[],Bt=He();Bt!==t;)ht.push(Bt),Bt=He();ht!==t?(Bt=jb(),Bt!==t?(Me=U,ce=ug(b,Se,Bt),U=ce):(Q=U,U=t)):(Q=U,U=t)}else Q=U,U=t;else Q=U,U=t;for(;U!==t;){for(N.push(U),U=Q,ce=[],Se=He();Se!==t;)ce.push(Se),Se=He();if(ce!==t)if(r.charCodeAt(Q)===43?(Se=Ms,Q++):(Se=t,I===0&&Qe(Dl)),Se===t&&(r.charCodeAt(Q)===45?(Se=fA,Q++):(Se=t,I===0&&Qe(Pl))),Se!==t){for(ht=[],Bt=He();Bt!==t;)ht.push(Bt),Bt=He();ht!==t?(Bt=jb(),Bt!==t?(Me=U,ce=ug(b,Se,Bt),U=ce):(Q=U,U=t)):(Q=U,U=t)}else Q=U,U=t;else Q=U,U=t}N!==t?(Me=m,b=Us(b,N),m=b):(Q=m,m=t)}else Q=m,m=t;return m}function w1(){var m,b,N,U,ce,Se;if(m=Q,r.substr(Q,3)===pA?(b=pA,Q+=3):(b=t,I===0&&Qe(R)),b!==t){for(N=[],U=He();U!==t;)N.push(U),U=He();if(N!==t)if(U=y1(),U!==t){for(ce=[],Se=He();Se!==t;)ce.push(Se),Se=He();ce!==t?(r.substr(Q,2)===q?(Se=q,Q+=2):(Se=t,I===0&&Qe(Ce)),Se!==t?(Me=m,b=Ke(U),m=b):(Q=m,m=t)):(Q=m,m=t)}else Q=m,m=t;else Q=m,m=t}else Q=m,m=t;return m}function B1(){var m,b,N,U;return m=Q,r.substr(Q,2)===Re?(b=Re,Q+=2):(b=t,I===0&&Qe(ze)),b!==t?(N=Mr(),N!==t?(r.charCodeAt(Q)===41?(U=O,Q++):(U=t,I===0&&Qe(F)),U!==t?(Me=m,b=dt(N),m=b):(Q=m,m=t)):(Q=m,m=t)):(Q=m,m=t),m}function qb(){var m,b,N,U,ce,Se;return m=Q,r.substr(Q,2)===Ft?(b=Ft,Q+=2):(b=t,I===0&&Qe(Fn)),b!==t?(N=Rl(),N!==t?(r.substr(Q,2)===Db?(U=Db,Q+=2):(U=t,I===0&&Qe($M)),U!==t?(ce=f1(),ce!==t?(r.charCodeAt(Q)===125?(Se=Fe,Q++):(Se=t,I===0&&Qe(Ne)),Se!==t?(Me=m,b=e1(N,ce),m=b):(Q=m,m=t)):(Q=m,m=t)):(Q=m,m=t)):(Q=m,m=t)):(Q=m,m=t),m===t&&(m=Q,r.substr(Q,2)===Ft?(b=Ft,Q+=2):(b=t,I===0&&Qe(Fn)),b!==t?(N=Rl(),N!==t?(r.substr(Q,3)===kb?(U=kb,Q+=3):(U=t,I===0&&Qe(t1)),U!==t?(Me=m,b=r1(N),m=b):(Q=m,m=t)):(Q=m,m=t)):(Q=m,m=t),m===t&&(m=Q,r.substr(Q,2)===Ft?(b=Ft,Q+=2):(b=t,I===0&&Qe(Fn)),b!==t?(N=Rl(),N!==t?(r.substr(Q,2)===Rb?(U=Rb,Q+=2):(U=t,I===0&&Qe(i1)),U!==t?(ce=f1(),ce!==t?(r.charCodeAt(Q)===125?(Se=Fe,Q++):(Se=t,I===0&&Qe(Ne)),Se!==t?(Me=m,b=n1(N,ce),m=b):(Q=m,m=t)):(Q=m,m=t)):(Q=m,m=t)):(Q=m,m=t)):(Q=m,m=t),m===t&&(m=Q,r.substr(Q,2)===Ft?(b=Ft,Q+=2):(b=t,I===0&&Qe(Fn)),b!==t?(N=Rl(),N!==t?(r.substr(Q,3)===Fb?(U=Fb,Q+=3):(U=t,I===0&&Qe(s1)),U!==t?(Me=m,b=o1(N),m=b):(Q=m,m=t)):(Q=m,m=t)):(Q=m,m=t),m===t&&(m=Q,r.substr(Q,2)===Ft?(b=Ft,Q+=2):(b=t,I===0&&Qe(Fn)),b!==t?(N=Rl(),N!==t?(r.charCodeAt(Q)===125?(U=Fe,Q++):(U=t,I===0&&Qe(Ne)),U!==t?(Me=m,b=Nb(N),m=b):(Q=m,m=t)):(Q=m,m=t)):(Q=m,m=t),m===t&&(m=Q,r.charCodeAt(Q)===36?(b=a1,Q++):(b=t,I===0&&Qe(A1)),b!==t?(N=Rl(),N!==t?(Me=m,b=Nb(N),m=b):(Q=m,m=t)):(Q=m,m=t)))))),m}function Tue(){var m,b,N;return m=Q,b=Oue(),b!==t?(Me=Q,N=l1(b),N?N=void 0:N=t,N!==t?(Me=m,b=c1(b),m=b):(Q=m,m=t)):(Q=m,m=t),m}function Oue(){var m,b,N,U,ce;if(m=Q,b=[],N=Q,U=Q,I++,ce=S1(),I--,ce===t?U=void 0:(Q=U,U=t),U!==t?(r.length>Q?(ce=r.charAt(Q),Q++):(ce=t,I===0&&Qe(Eo)),ce!==t?(Me=N,U=an(ce),N=U):(Q=N,N=t)):(Q=N,N=t),N!==t)for(;N!==t;)b.push(N),N=Q,U=Q,I++,ce=S1(),I--,ce===t?U=void 0:(Q=U,U=t),U!==t?(r.length>Q?(ce=r.charAt(Q),Q++):(ce=t,I===0&&Qe(Eo)),ce!==t?(Me=N,U=an(ce),N=U):(Q=N,N=t)):(Q=N,N=t);else b=t;return b!==t&&(Me=m,b=Dn(b)),m=b,m}function Q1(){var m,b,N;if(m=Q,b=[],Lb.test(r.charAt(Q))?(N=r.charAt(Q),Q++):(N=t,I===0&&Qe(Tb)),N!==t)for(;N!==t;)b.push(N),Lb.test(r.charAt(Q))?(N=r.charAt(Q),Q++):(N=t,I===0&&Qe(Tb));else b=t;return b!==t&&(Me=m,b=Ob()),m=b,m}function Rl(){var m,b,N;if(m=Q,b=[],Mb.test(r.charAt(Q))?(N=r.charAt(Q),Q++):(N=t,I===0&&Qe(Kb)),N!==t)for(;N!==t;)b.push(N),Mb.test(r.charAt(Q))?(N=r.charAt(Q),Q++):(N=t,I===0&&Qe(Kb));else b=t;return b!==t&&(Me=m,b=Ob()),m=b,m}function b1(){var m;return u1.test(r.charAt(Q))?(m=r.charAt(Q),Q++):(m=t,I===0&&Qe(gg)),m}function S1(){var m;return Ub.test(r.charAt(Q))?(m=r.charAt(Q),Q++):(m=t,I===0&&Qe(Hb)),m}function He(){var m,b;if(m=[],kE.test(r.charAt(Q))?(b=r.charAt(Q),Q++):(b=t,I===0&&Qe(RE)),b!==t)for(;b!==t;)m.push(b),kE.test(r.charAt(Q))?(b=r.charAt(Q),Q++):(b=t,I===0&&Qe(RE));else m=t;return m}if(k=n(),k!==t&&Q===r.length)return k;throw k!==t&&Q{"use strict";function Mge(r,e){function t(){this.constructor=r}t.prototype=e.prototype,r.prototype=new t}function Ul(r,e,t,i){this.message=r,this.expected=e,this.found=t,this.location=i,this.name="SyntaxError",typeof Error.captureStackTrace=="function"&&Error.captureStackTrace(this,Ul)}Mge(Ul,Error);Ul.buildMessage=function(r,e){var t={literal:function(c){return'"'+n(c.text)+'"'},class:function(c){var u="",g;for(g=0;g0){for(g=1,f=1;gH&&(H=v,j=[]),j.push(ne))}function Ne(ne,Y){return new Ul(ne,null,null,Y)}function oe(ne,Y,pe){return new Ul(Ul.buildMessage(ne,Y),ne,Y,pe)}function le(){var ne,Y,pe,ie;return ne=v,Y=we(),Y!==t?(r.charCodeAt(v)===47?(pe=s,v++):(pe=t,$===0&&Fe(o)),pe!==t?(ie=we(),ie!==t?(D=ne,Y=a(Y,ie),ne=Y):(v=ne,ne=t)):(v=ne,ne=t)):(v=ne,ne=t),ne===t&&(ne=v,Y=we(),Y!==t&&(D=ne,Y=l(Y)),ne=Y),ne}function we(){var ne,Y,pe,ie;return ne=v,Y=fe(),Y!==t?(r.charCodeAt(v)===64?(pe=c,v++):(pe=t,$===0&&Fe(u)),pe!==t?(ie=qe(),ie!==t?(D=ne,Y=g(Y,ie),ne=Y):(v=ne,ne=t)):(v=ne,ne=t)):(v=ne,ne=t),ne===t&&(ne=v,Y=fe(),Y!==t&&(D=ne,Y=f(Y)),ne=Y),ne}function fe(){var ne,Y,pe,ie,de;return ne=v,r.charCodeAt(v)===64?(Y=c,v++):(Y=t,$===0&&Fe(u)),Y!==t?(pe=Ae(),pe!==t?(r.charCodeAt(v)===47?(ie=s,v++):(ie=t,$===0&&Fe(o)),ie!==t?(de=Ae(),de!==t?(D=ne,Y=h(),ne=Y):(v=ne,ne=t)):(v=ne,ne=t)):(v=ne,ne=t)):(v=ne,ne=t),ne===t&&(ne=v,Y=Ae(),Y!==t&&(D=ne,Y=h()),ne=Y),ne}function Ae(){var ne,Y,pe;if(ne=v,Y=[],p.test(r.charAt(v))?(pe=r.charAt(v),v++):(pe=t,$===0&&Fe(C)),pe!==t)for(;pe!==t;)Y.push(pe),p.test(r.charAt(v))?(pe=r.charAt(v),v++):(pe=t,$===0&&Fe(C));else Y=t;return Y!==t&&(D=ne,Y=h()),ne=Y,ne}function qe(){var ne,Y,pe;if(ne=v,Y=[],y.test(r.charAt(v))?(pe=r.charAt(v),v++):(pe=t,$===0&&Fe(B)),pe!==t)for(;pe!==t;)Y.push(pe),y.test(r.charAt(v))?(pe=r.charAt(v),v++):(pe=t,$===0&&Fe(B));else Y=t;return Y!==t&&(D=ne,Y=h()),ne=Y,ne}if(V=n(),V!==t&&v===r.length)return V;throw V!==t&&v{"use strict";function UK(r){return typeof r>"u"||r===null}function Uge(r){return typeof r=="object"&&r!==null}function Hge(r){return Array.isArray(r)?r:UK(r)?[]:[r]}function Gge(r,e){var t,i,n,s;if(e)for(s=Object.keys(e),t=0,i=s.length;t{"use strict";function Op(r,e){Error.call(this),this.name="YAMLException",this.reason=r,this.mark=e,this.message=(this.reason||"(unknown reason)")+(this.mark?" "+this.mark.toString():""),Error.captureStackTrace?Error.captureStackTrace(this,this.constructor):this.stack=new Error().stack||""}Op.prototype=Object.create(Error.prototype);Op.prototype.constructor=Op;Op.prototype.toString=function(e){var t=this.name+": ";return t+=this.reason||"(unknown reason)",!e&&this.mark&&(t+=" "+this.mark.toString()),t};HK.exports=Op});var jK=w((pXe,YK)=>{"use strict";var GK=Gl();function SS(r,e,t,i,n){this.name=r,this.buffer=e,this.position=t,this.line=i,this.column=n}SS.prototype.getSnippet=function(e,t){var i,n,s,o,a;if(!this.buffer)return null;for(e=e||4,t=t||75,i="",n=this.position;n>0&&`\0\r -\x85\u2028\u2029`.indexOf(this.buffer.charAt(n-1))===-1;)if(n-=1,this.position-n>t/2-1){i=" ... ",n+=5;break}for(s="",o=this.position;ot/2-1){s=" ... ",o-=5;break}return a=this.buffer.slice(n,o),GK.repeat(" ",e)+i+a+s+` -`+GK.repeat(" ",e+this.position-n+i.length)+"^"};SS.prototype.toString=function(e){var t,i="";return this.name&&(i+='in "'+this.name+'" '),i+="at line "+(this.line+1)+", column "+(this.column+1),e||(t=this.getSnippet(),t&&(i+=`: -`+t)),i};YK.exports=SS});var si=w((dXe,JK)=>{"use strict";var qK=Qg(),qge=["kind","resolve","construct","instanceOf","predicate","represent","defaultStyle","styleAliases"],Jge=["scalar","sequence","mapping"];function Wge(r){var e={};return r!==null&&Object.keys(r).forEach(function(t){r[t].forEach(function(i){e[String(i)]=t})}),e}function zge(r,e){if(e=e||{},Object.keys(e).forEach(function(t){if(qge.indexOf(t)===-1)throw new qK('Unknown option "'+t+'" is met in definition of "'+r+'" YAML type.')}),this.tag=r,this.kind=e.kind||null,this.resolve=e.resolve||function(){return!0},this.construct=e.construct||function(t){return t},this.instanceOf=e.instanceOf||null,this.predicate=e.predicate||null,this.represent=e.represent||null,this.defaultStyle=e.defaultStyle||null,this.styleAliases=Wge(e.styleAliases||null),Jge.indexOf(this.kind)===-1)throw new qK('Unknown kind "'+this.kind+'" is specified for "'+r+'" YAML type.')}JK.exports=zge});var Yl=w((CXe,zK)=>{"use strict";var WK=Gl(),nI=Qg(),Vge=si();function vS(r,e,t){var i=[];return r.include.forEach(function(n){t=vS(n,e,t)}),r[e].forEach(function(n){t.forEach(function(s,o){s.tag===n.tag&&s.kind===n.kind&&i.push(o)}),t.push(n)}),t.filter(function(n,s){return i.indexOf(s)===-1})}function Xge(){var r={scalar:{},sequence:{},mapping:{},fallback:{}},e,t;function i(n){r[n.kind][n.tag]=r.fallback[n.tag]=n}for(e=0,t=arguments.length;e{"use strict";var _ge=si();VK.exports=new _ge("tag:yaml.org,2002:str",{kind:"scalar",construct:function(r){return r!==null?r:""}})});var ZK=w((EXe,_K)=>{"use strict";var Zge=si();_K.exports=new Zge("tag:yaml.org,2002:seq",{kind:"sequence",construct:function(r){return r!==null?r:[]}})});var eU=w((IXe,$K)=>{"use strict";var $ge=si();$K.exports=new $ge("tag:yaml.org,2002:map",{kind:"mapping",construct:function(r){return r!==null?r:{}}})});var sI=w((yXe,tU)=>{"use strict";var efe=Yl();tU.exports=new efe({explicit:[XK(),ZK(),eU()]})});var iU=w((wXe,rU)=>{"use strict";var tfe=si();function rfe(r){if(r===null)return!0;var e=r.length;return e===1&&r==="~"||e===4&&(r==="null"||r==="Null"||r==="NULL")}function ife(){return null}function nfe(r){return r===null}rU.exports=new tfe("tag:yaml.org,2002:null",{kind:"scalar",resolve:rfe,construct:ife,predicate:nfe,represent:{canonical:function(){return"~"},lowercase:function(){return"null"},uppercase:function(){return"NULL"},camelcase:function(){return"Null"}},defaultStyle:"lowercase"})});var sU=w((BXe,nU)=>{"use strict";var sfe=si();function ofe(r){if(r===null)return!1;var e=r.length;return e===4&&(r==="true"||r==="True"||r==="TRUE")||e===5&&(r==="false"||r==="False"||r==="FALSE")}function afe(r){return r==="true"||r==="True"||r==="TRUE"}function Afe(r){return Object.prototype.toString.call(r)==="[object Boolean]"}nU.exports=new sfe("tag:yaml.org,2002:bool",{kind:"scalar",resolve:ofe,construct:afe,predicate:Afe,represent:{lowercase:function(r){return r?"true":"false"},uppercase:function(r){return r?"TRUE":"FALSE"},camelcase:function(r){return r?"True":"False"}},defaultStyle:"lowercase"})});var aU=w((QXe,oU)=>{"use strict";var lfe=Gl(),cfe=si();function ufe(r){return 48<=r&&r<=57||65<=r&&r<=70||97<=r&&r<=102}function gfe(r){return 48<=r&&r<=55}function ffe(r){return 48<=r&&r<=57}function hfe(r){if(r===null)return!1;var e=r.length,t=0,i=!1,n;if(!e)return!1;if(n=r[t],(n==="-"||n==="+")&&(n=r[++t]),n==="0"){if(t+1===e)return!0;if(n=r[++t],n==="b"){for(t++;t=0?"0b"+r.toString(2):"-0b"+r.toString(2).slice(1)},octal:function(r){return r>=0?"0"+r.toString(8):"-0"+r.toString(8).slice(1)},decimal:function(r){return r.toString(10)},hexadecimal:function(r){return r>=0?"0x"+r.toString(16).toUpperCase():"-0x"+r.toString(16).toUpperCase().slice(1)}},defaultStyle:"decimal",styleAliases:{binary:[2,"bin"],octal:[8,"oct"],decimal:[10,"dec"],hexadecimal:[16,"hex"]}})});var cU=w((bXe,lU)=>{"use strict";var AU=Gl(),Cfe=si(),mfe=new RegExp("^(?:[-+]?(?:0|[1-9][0-9_]*)(?:\\.[0-9_]*)?(?:[eE][-+]?[0-9]+)?|\\.[0-9_]+(?:[eE][-+]?[0-9]+)?|[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\\.[0-9_]*|[-+]?\\.(?:inf|Inf|INF)|\\.(?:nan|NaN|NAN))$");function Efe(r){return!(r===null||!mfe.test(r)||r[r.length-1]==="_")}function Ife(r){var e,t,i,n;return e=r.replace(/_/g,"").toLowerCase(),t=e[0]==="-"?-1:1,n=[],"+-".indexOf(e[0])>=0&&(e=e.slice(1)),e===".inf"?t===1?Number.POSITIVE_INFINITY:Number.NEGATIVE_INFINITY:e===".nan"?NaN:e.indexOf(":")>=0?(e.split(":").forEach(function(s){n.unshift(parseFloat(s,10))}),e=0,i=1,n.forEach(function(s){e+=s*i,i*=60}),t*e):t*parseFloat(e,10)}var yfe=/^[-+]?[0-9]+e/;function wfe(r,e){var t;if(isNaN(r))switch(e){case"lowercase":return".nan";case"uppercase":return".NAN";case"camelcase":return".NaN"}else if(Number.POSITIVE_INFINITY===r)switch(e){case"lowercase":return".inf";case"uppercase":return".INF";case"camelcase":return".Inf"}else if(Number.NEGATIVE_INFINITY===r)switch(e){case"lowercase":return"-.inf";case"uppercase":return"-.INF";case"camelcase":return"-.Inf"}else if(AU.isNegativeZero(r))return"-0.0";return t=r.toString(10),yfe.test(t)?t.replace("e",".e"):t}function Bfe(r){return Object.prototype.toString.call(r)==="[object Number]"&&(r%1!==0||AU.isNegativeZero(r))}lU.exports=new Cfe("tag:yaml.org,2002:float",{kind:"scalar",resolve:Efe,construct:Ife,predicate:Bfe,represent:wfe,defaultStyle:"lowercase"})});var xS=w((SXe,uU)=>{"use strict";var Qfe=Yl();uU.exports=new Qfe({include:[sI()],implicit:[iU(),sU(),aU(),cU()]})});var PS=w((vXe,gU)=>{"use strict";var bfe=Yl();gU.exports=new bfe({include:[xS()]})});var dU=w((xXe,pU)=>{"use strict";var Sfe=si(),fU=new RegExp("^([0-9][0-9][0-9][0-9])-([0-9][0-9])-([0-9][0-9])$"),hU=new RegExp("^([0-9][0-9][0-9][0-9])-([0-9][0-9]?)-([0-9][0-9]?)(?:[Tt]|[ \\t]+)([0-9][0-9]?):([0-9][0-9]):([0-9][0-9])(?:\\.([0-9]*))?(?:[ \\t]*(Z|([-+])([0-9][0-9]?)(?::([0-9][0-9]))?))?$");function vfe(r){return r===null?!1:fU.exec(r)!==null||hU.exec(r)!==null}function xfe(r){var e,t,i,n,s,o,a,l=0,c=null,u,g,f;if(e=fU.exec(r),e===null&&(e=hU.exec(r)),e===null)throw new Error("Date resolve error");if(t=+e[1],i=+e[2]-1,n=+e[3],!e[4])return new Date(Date.UTC(t,i,n));if(s=+e[4],o=+e[5],a=+e[6],e[7]){for(l=e[7].slice(0,3);l.length<3;)l+="0";l=+l}return e[9]&&(u=+e[10],g=+(e[11]||0),c=(u*60+g)*6e4,e[9]==="-"&&(c=-c)),f=new Date(Date.UTC(t,i,n,s,o,a,l)),c&&f.setTime(f.getTime()-c),f}function Pfe(r){return r.toISOString()}pU.exports=new Sfe("tag:yaml.org,2002:timestamp",{kind:"scalar",resolve:vfe,construct:xfe,instanceOf:Date,represent:Pfe})});var mU=w((PXe,CU)=>{"use strict";var Dfe=si();function kfe(r){return r==="<<"||r===null}CU.exports=new Dfe("tag:yaml.org,2002:merge",{kind:"scalar",resolve:kfe})});var yU=w((DXe,IU)=>{"use strict";var jl;try{EU=J,jl=EU("buffer").Buffer}catch{}var EU,Rfe=si(),DS=`ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/= -\r`;function Ffe(r){if(r===null)return!1;var e,t,i=0,n=r.length,s=DS;for(t=0;t64)){if(e<0)return!1;i+=6}return i%8===0}function Nfe(r){var e,t,i=r.replace(/[\r\n=]/g,""),n=i.length,s=DS,o=0,a=[];for(e=0;e>16&255),a.push(o>>8&255),a.push(o&255)),o=o<<6|s.indexOf(i.charAt(e));return t=n%4*6,t===0?(a.push(o>>16&255),a.push(o>>8&255),a.push(o&255)):t===18?(a.push(o>>10&255),a.push(o>>2&255)):t===12&&a.push(o>>4&255),jl?jl.from?jl.from(a):new jl(a):a}function Lfe(r){var e="",t=0,i,n,s=r.length,o=DS;for(i=0;i>18&63],e+=o[t>>12&63],e+=o[t>>6&63],e+=o[t&63]),t=(t<<8)+r[i];return n=s%3,n===0?(e+=o[t>>18&63],e+=o[t>>12&63],e+=o[t>>6&63],e+=o[t&63]):n===2?(e+=o[t>>10&63],e+=o[t>>4&63],e+=o[t<<2&63],e+=o[64]):n===1&&(e+=o[t>>2&63],e+=o[t<<4&63],e+=o[64],e+=o[64]),e}function Tfe(r){return jl&&jl.isBuffer(r)}IU.exports=new Rfe("tag:yaml.org,2002:binary",{kind:"scalar",resolve:Ffe,construct:Nfe,predicate:Tfe,represent:Lfe})});var BU=w((kXe,wU)=>{"use strict";var Ofe=si(),Mfe=Object.prototype.hasOwnProperty,Kfe=Object.prototype.toString;function Ufe(r){if(r===null)return!0;var e=[],t,i,n,s,o,a=r;for(t=0,i=a.length;t{"use strict";var Gfe=si(),Yfe=Object.prototype.toString;function jfe(r){if(r===null)return!0;var e,t,i,n,s,o=r;for(s=new Array(o.length),e=0,t=o.length;e{"use strict";var Jfe=si(),Wfe=Object.prototype.hasOwnProperty;function zfe(r){if(r===null)return!0;var e,t=r;for(e in t)if(Wfe.call(t,e)&&t[e]!==null)return!1;return!0}function Vfe(r){return r!==null?r:{}}SU.exports=new Jfe("tag:yaml.org,2002:set",{kind:"mapping",resolve:zfe,construct:Vfe})});var Sg=w((NXe,xU)=>{"use strict";var Xfe=Yl();xU.exports=new Xfe({include:[PS()],implicit:[dU(),mU()],explicit:[yU(),BU(),bU(),vU()]})});var DU=w((LXe,PU)=>{"use strict";var _fe=si();function Zfe(){return!0}function $fe(){}function ehe(){return""}function the(r){return typeof r>"u"}PU.exports=new _fe("tag:yaml.org,2002:js/undefined",{kind:"scalar",resolve:Zfe,construct:$fe,predicate:the,represent:ehe})});var RU=w((TXe,kU)=>{"use strict";var rhe=si();function ihe(r){if(r===null||r.length===0)return!1;var e=r,t=/\/([gim]*)$/.exec(r),i="";return!(e[0]==="/"&&(t&&(i=t[1]),i.length>3||e[e.length-i.length-1]!=="/"))}function nhe(r){var e=r,t=/\/([gim]*)$/.exec(r),i="";return e[0]==="/"&&(t&&(i=t[1]),e=e.slice(1,e.length-i.length-1)),new RegExp(e,i)}function she(r){var e="/"+r.source+"/";return r.global&&(e+="g"),r.multiline&&(e+="m"),r.ignoreCase&&(e+="i"),e}function ohe(r){return Object.prototype.toString.call(r)==="[object RegExp]"}kU.exports=new rhe("tag:yaml.org,2002:js/regexp",{kind:"scalar",resolve:ihe,construct:nhe,predicate:ohe,represent:she})});var LU=w((OXe,NU)=>{"use strict";var oI;try{FU=J,oI=FU("esprima")}catch{typeof window<"u"&&(oI=window.esprima)}var FU,ahe=si();function Ahe(r){if(r===null)return!1;try{var e="("+r+")",t=oI.parse(e,{range:!0});return!(t.type!=="Program"||t.body.length!==1||t.body[0].type!=="ExpressionStatement"||t.body[0].expression.type!=="ArrowFunctionExpression"&&t.body[0].expression.type!=="FunctionExpression")}catch{return!1}}function lhe(r){var e="("+r+")",t=oI.parse(e,{range:!0}),i=[],n;if(t.type!=="Program"||t.body.length!==1||t.body[0].type!=="ExpressionStatement"||t.body[0].expression.type!=="ArrowFunctionExpression"&&t.body[0].expression.type!=="FunctionExpression")throw new Error("Failed to resolve function");return t.body[0].expression.params.forEach(function(s){i.push(s.name)}),n=t.body[0].expression.body.range,t.body[0].expression.body.type==="BlockStatement"?new Function(i,e.slice(n[0]+1,n[1]-1)):new Function(i,"return "+e.slice(n[0],n[1]))}function che(r){return r.toString()}function uhe(r){return Object.prototype.toString.call(r)==="[object Function]"}NU.exports=new ahe("tag:yaml.org,2002:js/function",{kind:"scalar",resolve:Ahe,construct:lhe,predicate:uhe,represent:che})});var Mp=w((MXe,OU)=>{"use strict";var TU=Yl();OU.exports=TU.DEFAULT=new TU({include:[Sg()],explicit:[DU(),RU(),LU()]})});var r2=w((KXe,Kp)=>{"use strict";var da=Gl(),jU=Qg(),ghe=jK(),qU=Sg(),fhe=Mp(),wA=Object.prototype.hasOwnProperty,aI=1,JU=2,WU=3,AI=4,kS=1,hhe=2,MU=3,phe=/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F-\x84\x86-\x9F\uFFFE\uFFFF]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF]/,dhe=/[\x85\u2028\u2029]/,Che=/[,\[\]\{\}]/,zU=/^(?:!|!!|![a-z\-]+!)$/i,VU=/^(?:!|[^,\[\]\{\}])(?:%[0-9a-f]{2}|[0-9a-z\-#;\/\?:@&=\+\$,_\.!~\*'\(\)\[\]])*$/i;function KU(r){return Object.prototype.toString.call(r)}function Bo(r){return r===10||r===13}function Jl(r){return r===9||r===32}function un(r){return r===9||r===32||r===10||r===13}function vg(r){return r===44||r===91||r===93||r===123||r===125}function mhe(r){var e;return 48<=r&&r<=57?r-48:(e=r|32,97<=e&&e<=102?e-97+10:-1)}function Ehe(r){return r===120?2:r===117?4:r===85?8:0}function Ihe(r){return 48<=r&&r<=57?r-48:-1}function UU(r){return r===48?"\0":r===97?"\x07":r===98?"\b":r===116||r===9?" ":r===110?` -`:r===118?"\v":r===102?"\f":r===114?"\r":r===101?"\x1B":r===32?" ":r===34?'"':r===47?"/":r===92?"\\":r===78?"\x85":r===95?"\xA0":r===76?"\u2028":r===80?"\u2029":""}function yhe(r){return r<=65535?String.fromCharCode(r):String.fromCharCode((r-65536>>10)+55296,(r-65536&1023)+56320)}var XU=new Array(256),_U=new Array(256);for(ql=0;ql<256;ql++)XU[ql]=UU(ql)?1:0,_U[ql]=UU(ql);var ql;function whe(r,e){this.input=r,this.filename=e.filename||null,this.schema=e.schema||fhe,this.onWarning=e.onWarning||null,this.legacy=e.legacy||!1,this.json=e.json||!1,this.listener=e.listener||null,this.implicitTypes=this.schema.compiledImplicit,this.typeMap=this.schema.compiledTypeMap,this.length=r.length,this.position=0,this.line=0,this.lineStart=0,this.lineIndent=0,this.documents=[]}function ZU(r,e){return new jU(e,new ghe(r.filename,r.input,r.position,r.line,r.position-r.lineStart))}function ft(r,e){throw ZU(r,e)}function lI(r,e){r.onWarning&&r.onWarning.call(null,ZU(r,e))}var HU={YAML:function(e,t,i){var n,s,o;e.version!==null&&ft(e,"duplication of %YAML directive"),i.length!==1&&ft(e,"YAML directive accepts exactly one argument"),n=/^([0-9]+)\.([0-9]+)$/.exec(i[0]),n===null&&ft(e,"ill-formed argument of the YAML directive"),s=parseInt(n[1],10),o=parseInt(n[2],10),s!==1&&ft(e,"unacceptable YAML version of the document"),e.version=i[0],e.checkLineBreaks=o<2,o!==1&&o!==2&&lI(e,"unsupported YAML version of the document")},TAG:function(e,t,i){var n,s;i.length!==2&&ft(e,"TAG directive accepts exactly two arguments"),n=i[0],s=i[1],zU.test(n)||ft(e,"ill-formed tag handle (first argument) of the TAG directive"),wA.call(e.tagMap,n)&&ft(e,'there is a previously declared suffix for "'+n+'" tag handle'),VU.test(s)||ft(e,"ill-formed tag prefix (second argument) of the TAG directive"),e.tagMap[n]=s}};function yA(r,e,t,i){var n,s,o,a;if(e1&&(r.result+=da.repeat(` -`,e-1))}function Bhe(r,e,t){var i,n,s,o,a,l,c,u,g=r.kind,f=r.result,h;if(h=r.input.charCodeAt(r.position),un(h)||vg(h)||h===35||h===38||h===42||h===33||h===124||h===62||h===39||h===34||h===37||h===64||h===96||(h===63||h===45)&&(n=r.input.charCodeAt(r.position+1),un(n)||t&&vg(n)))return!1;for(r.kind="scalar",r.result="",s=o=r.position,a=!1;h!==0;){if(h===58){if(n=r.input.charCodeAt(r.position+1),un(n)||t&&vg(n))break}else if(h===35){if(i=r.input.charCodeAt(r.position-1),un(i))break}else{if(r.position===r.lineStart&&cI(r)||t&&vg(h))break;if(Bo(h))if(l=r.line,c=r.lineStart,u=r.lineIndent,zr(r,!1,-1),r.lineIndent>=e){a=!0,h=r.input.charCodeAt(r.position);continue}else{r.position=o,r.line=l,r.lineStart=c,r.lineIndent=u;break}}a&&(yA(r,s,o,!1),FS(r,r.line-l),s=o=r.position,a=!1),Jl(h)||(o=r.position+1),h=r.input.charCodeAt(++r.position)}return yA(r,s,o,!1),r.result?!0:(r.kind=g,r.result=f,!1)}function Qhe(r,e){var t,i,n;if(t=r.input.charCodeAt(r.position),t!==39)return!1;for(r.kind="scalar",r.result="",r.position++,i=n=r.position;(t=r.input.charCodeAt(r.position))!==0;)if(t===39)if(yA(r,i,r.position,!0),t=r.input.charCodeAt(++r.position),t===39)i=r.position,r.position++,n=r.position;else return!0;else Bo(t)?(yA(r,i,n,!0),FS(r,zr(r,!1,e)),i=n=r.position):r.position===r.lineStart&&cI(r)?ft(r,"unexpected end of the document within a single quoted scalar"):(r.position++,n=r.position);ft(r,"unexpected end of the stream within a single quoted scalar")}function bhe(r,e){var t,i,n,s,o,a;if(a=r.input.charCodeAt(r.position),a!==34)return!1;for(r.kind="scalar",r.result="",r.position++,t=i=r.position;(a=r.input.charCodeAt(r.position))!==0;){if(a===34)return yA(r,t,r.position,!0),r.position++,!0;if(a===92){if(yA(r,t,r.position,!0),a=r.input.charCodeAt(++r.position),Bo(a))zr(r,!1,e);else if(a<256&&XU[a])r.result+=_U[a],r.position++;else if((o=Ehe(a))>0){for(n=o,s=0;n>0;n--)a=r.input.charCodeAt(++r.position),(o=mhe(a))>=0?s=(s<<4)+o:ft(r,"expected hexadecimal character");r.result+=yhe(s),r.position++}else ft(r,"unknown escape sequence");t=i=r.position}else Bo(a)?(yA(r,t,i,!0),FS(r,zr(r,!1,e)),t=i=r.position):r.position===r.lineStart&&cI(r)?ft(r,"unexpected end of the document within a double quoted scalar"):(r.position++,i=r.position)}ft(r,"unexpected end of the stream within a double quoted scalar")}function She(r,e){var t=!0,i,n=r.tag,s,o=r.anchor,a,l,c,u,g,f={},h,p,C,y;if(y=r.input.charCodeAt(r.position),y===91)l=93,g=!1,s=[];else if(y===123)l=125,g=!0,s={};else return!1;for(r.anchor!==null&&(r.anchorMap[r.anchor]=s),y=r.input.charCodeAt(++r.position);y!==0;){if(zr(r,!0,e),y=r.input.charCodeAt(r.position),y===l)return r.position++,r.tag=n,r.anchor=o,r.kind=g?"mapping":"sequence",r.result=s,!0;t||ft(r,"missed comma between flow collection entries"),p=h=C=null,c=u=!1,y===63&&(a=r.input.charCodeAt(r.position+1),un(a)&&(c=u=!0,r.position++,zr(r,!0,e))),i=r.line,Pg(r,e,aI,!1,!0),p=r.tag,h=r.result,zr(r,!0,e),y=r.input.charCodeAt(r.position),(u||r.line===i)&&y===58&&(c=!0,y=r.input.charCodeAt(++r.position),zr(r,!0,e),Pg(r,e,aI,!1,!0),C=r.result),g?xg(r,s,f,p,h,C):c?s.push(xg(r,null,f,p,h,C)):s.push(h),zr(r,!0,e),y=r.input.charCodeAt(r.position),y===44?(t=!0,y=r.input.charCodeAt(++r.position)):t=!1}ft(r,"unexpected end of the stream within a flow collection")}function vhe(r,e){var t,i,n=kS,s=!1,o=!1,a=e,l=0,c=!1,u,g;if(g=r.input.charCodeAt(r.position),g===124)i=!1;else if(g===62)i=!0;else return!1;for(r.kind="scalar",r.result="";g!==0;)if(g=r.input.charCodeAt(++r.position),g===43||g===45)kS===n?n=g===43?MU:hhe:ft(r,"repeat of a chomping mode identifier");else if((u=Ihe(g))>=0)u===0?ft(r,"bad explicit indentation width of a block scalar; it cannot be less than one"):o?ft(r,"repeat of an indentation width identifier"):(a=e+u-1,o=!0);else break;if(Jl(g)){do g=r.input.charCodeAt(++r.position);while(Jl(g));if(g===35)do g=r.input.charCodeAt(++r.position);while(!Bo(g)&&g!==0)}for(;g!==0;){for(RS(r),r.lineIndent=0,g=r.input.charCodeAt(r.position);(!o||r.lineIndenta&&(a=r.lineIndent),Bo(g)){l++;continue}if(r.lineIndente)&&l!==0)ft(r,"bad indentation of a sequence entry");else if(r.lineIndente)&&(Pg(r,e,AI,!0,n)&&(p?f=r.result:h=r.result),p||(xg(r,c,u,g,f,h,s,o),g=f=h=null),zr(r,!0,-1),y=r.input.charCodeAt(r.position)),r.lineIndent>e&&y!==0)ft(r,"bad indentation of a mapping entry");else if(r.lineIndente?l=1:r.lineIndent===e?l=0:r.lineIndente?l=1:r.lineIndent===e?l=0:r.lineIndent tag; it should be "scalar", not "'+r.kind+'"'),g=0,f=r.implicitTypes.length;g tag; it should be "'+h.kind+'", not "'+r.kind+'"'),h.resolve(r.result)?(r.result=h.construct(r.result),r.anchor!==null&&(r.anchorMap[r.anchor]=r.result)):ft(r,"cannot resolve a node with !<"+r.tag+"> explicit tag")):ft(r,"unknown tag !<"+r.tag+">");return r.listener!==null&&r.listener("close",r),r.tag!==null||r.anchor!==null||u}function Rhe(r){var e=r.position,t,i,n,s=!1,o;for(r.version=null,r.checkLineBreaks=r.legacy,r.tagMap={},r.anchorMap={};(o=r.input.charCodeAt(r.position))!==0&&(zr(r,!0,-1),o=r.input.charCodeAt(r.position),!(r.lineIndent>0||o!==37));){for(s=!0,o=r.input.charCodeAt(++r.position),t=r.position;o!==0&&!un(o);)o=r.input.charCodeAt(++r.position);for(i=r.input.slice(t,r.position),n=[],i.length<1&&ft(r,"directive name must not be less than one character in length");o!==0;){for(;Jl(o);)o=r.input.charCodeAt(++r.position);if(o===35){do o=r.input.charCodeAt(++r.position);while(o!==0&&!Bo(o));break}if(Bo(o))break;for(t=r.position;o!==0&&!un(o);)o=r.input.charCodeAt(++r.position);n.push(r.input.slice(t,r.position))}o!==0&&RS(r),wA.call(HU,i)?HU[i](r,i,n):lI(r,'unknown document directive "'+i+'"')}if(zr(r,!0,-1),r.lineIndent===0&&r.input.charCodeAt(r.position)===45&&r.input.charCodeAt(r.position+1)===45&&r.input.charCodeAt(r.position+2)===45?(r.position+=3,zr(r,!0,-1)):s&&ft(r,"directives end mark is expected"),Pg(r,r.lineIndent-1,AI,!1,!0),zr(r,!0,-1),r.checkLineBreaks&&dhe.test(r.input.slice(e,r.position))&&lI(r,"non-ASCII line breaks are interpreted as content"),r.documents.push(r.result),r.position===r.lineStart&&cI(r)){r.input.charCodeAt(r.position)===46&&(r.position+=3,zr(r,!0,-1));return}if(r.position"u"&&(t=e,e=null);var i=$U(r,t);if(typeof e!="function")return i;for(var n=0,s=i.length;n"u"&&(t=e,e=null),e2(r,e,da.extend({schema:qU},t))}function Nhe(r,e){return t2(r,da.extend({schema:qU},e))}Kp.exports.loadAll=e2;Kp.exports.load=t2;Kp.exports.safeLoadAll=Fhe;Kp.exports.safeLoad=Nhe});var b2=w((UXe,OS)=>{"use strict";var Hp=Gl(),Gp=Qg(),Lhe=Mp(),The=Sg(),c2=Object.prototype.toString,u2=Object.prototype.hasOwnProperty,Ohe=9,Up=10,Mhe=13,Khe=32,Uhe=33,Hhe=34,g2=35,Ghe=37,Yhe=38,jhe=39,qhe=42,f2=44,Jhe=45,h2=58,Whe=61,zhe=62,Vhe=63,Xhe=64,p2=91,d2=93,_he=96,C2=123,Zhe=124,m2=125,Fi={};Fi[0]="\\0";Fi[7]="\\a";Fi[8]="\\b";Fi[9]="\\t";Fi[10]="\\n";Fi[11]="\\v";Fi[12]="\\f";Fi[13]="\\r";Fi[27]="\\e";Fi[34]='\\"';Fi[92]="\\\\";Fi[133]="\\N";Fi[160]="\\_";Fi[8232]="\\L";Fi[8233]="\\P";var $he=["y","Y","yes","Yes","YES","on","On","ON","n","N","no","No","NO","off","Off","OFF"];function epe(r,e){var t,i,n,s,o,a,l;if(e===null)return{};for(t={},i=Object.keys(e),n=0,s=i.length;n0?r.charCodeAt(s-1):null,f=f&&s2(o,a)}else{for(s=0;si&&r[g+1]!==" ",g=s);else if(!Dg(o))return uI;a=s>0?r.charCodeAt(s-1):null,f=f&&s2(o,a)}c=c||u&&s-g-1>i&&r[g+1]!==" "}return!l&&!c?f&&!n(r)?I2:y2:t>9&&E2(r)?uI:c?B2:w2}function ope(r,e,t,i){r.dump=function(){if(e.length===0)return"''";if(!r.noCompatMode&&$he.indexOf(e)!==-1)return"'"+e+"'";var n=r.indent*Math.max(1,t),s=r.lineWidth===-1?-1:Math.max(Math.min(r.lineWidth,40),r.lineWidth-n),o=i||r.flowLevel>-1&&t>=r.flowLevel;function a(l){return rpe(r,l)}switch(spe(e,o,r.indent,s,a)){case I2:return e;case y2:return"'"+e.replace(/'/g,"''")+"'";case w2:return"|"+o2(e,r.indent)+a2(n2(e,n));case B2:return">"+o2(e,r.indent)+a2(n2(ape(e,s),n));case uI:return'"'+Ape(e,s)+'"';default:throw new Gp("impossible error: invalid scalar style")}}()}function o2(r,e){var t=E2(r)?String(e):"",i=r[r.length-1]===` -`,n=i&&(r[r.length-2]===` -`||r===` -`),s=n?"+":i?"":"-";return t+s+` -`}function a2(r){return r[r.length-1]===` -`?r.slice(0,-1):r}function ape(r,e){for(var t=/(\n+)([^\n]*)/g,i=function(){var c=r.indexOf(` -`);return c=c!==-1?c:r.length,t.lastIndex=c,A2(r.slice(0,c),e)}(),n=r[0]===` -`||r[0]===" ",s,o;o=t.exec(r);){var a=o[1],l=o[2];s=l[0]===" ",i+=a+(!n&&!s&&l!==""?` -`:"")+A2(l,e),n=s}return i}function A2(r,e){if(r===""||r[0]===" ")return r;for(var t=/ [^ ]/g,i,n=0,s,o=0,a=0,l="";i=t.exec(r);)a=i.index,a-n>e&&(s=o>n?o:a,l+=` -`+r.slice(n,s),n=s+1),o=a;return l+=` -`,r.length-n>e&&o>n?l+=r.slice(n,o)+` -`+r.slice(o+1):l+=r.slice(n),l.slice(1)}function Ape(r){for(var e="",t,i,n,s=0;s=55296&&t<=56319&&(i=r.charCodeAt(s+1),i>=56320&&i<=57343)){e+=i2((t-55296)*1024+i-56320+65536),s++;continue}n=Fi[t],e+=!n&&Dg(t)?r[s]:n||i2(t)}return e}function lpe(r,e,t){var i="",n=r.tag,s,o;for(s=0,o=t.length;s1024&&(u+="? "),u+=r.dump+(r.condenseFlow?'"':"")+":"+(r.condenseFlow?"":" "),Wl(r,e,c,!1,!1)&&(u+=r.dump,i+=u));r.tag=n,r.dump="{"+i+"}"}function gpe(r,e,t,i){var n="",s=r.tag,o=Object.keys(t),a,l,c,u,g,f;if(r.sortKeys===!0)o.sort();else if(typeof r.sortKeys=="function")o.sort(r.sortKeys);else if(r.sortKeys)throw new Gp("sortKeys must be a boolean or a function");for(a=0,l=o.length;a1024,g&&(r.dump&&Up===r.dump.charCodeAt(0)?f+="?":f+="? "),f+=r.dump,g&&(f+=NS(r,e)),Wl(r,e+1,u,!0,g)&&(r.dump&&Up===r.dump.charCodeAt(0)?f+=":":f+=": ",f+=r.dump,n+=f));r.tag=s,r.dump=n||"{}"}function l2(r,e,t){var i,n,s,o,a,l;for(n=t?r.explicitTypes:r.implicitTypes,s=0,o=n.length;s tag resolver accepts not "'+l+'" style');r.dump=i}return!0}return!1}function Wl(r,e,t,i,n,s){r.tag=null,r.dump=t,l2(r,t,!1)||l2(r,t,!0);var o=c2.call(r.dump);i&&(i=r.flowLevel<0||r.flowLevel>e);var a=o==="[object Object]"||o==="[object Array]",l,c;if(a&&(l=r.duplicates.indexOf(t),c=l!==-1),(r.tag!==null&&r.tag!=="?"||c||r.indent!==2&&e>0)&&(n=!1),c&&r.usedDuplicates[l])r.dump="*ref_"+l;else{if(a&&c&&!r.usedDuplicates[l]&&(r.usedDuplicates[l]=!0),o==="[object Object]")i&&Object.keys(r.dump).length!==0?(gpe(r,e,r.dump,n),c&&(r.dump="&ref_"+l+r.dump)):(upe(r,e,r.dump),c&&(r.dump="&ref_"+l+" "+r.dump));else if(o==="[object Array]"){var u=r.noArrayIndent&&e>0?e-1:e;i&&r.dump.length!==0?(cpe(r,u,r.dump,n),c&&(r.dump="&ref_"+l+r.dump)):(lpe(r,u,r.dump),c&&(r.dump="&ref_"+l+" "+r.dump))}else if(o==="[object String]")r.tag!=="?"&&ope(r,r.dump,e,s);else{if(r.skipInvalid)return!1;throw new Gp("unacceptable kind of an object to dump "+o)}r.tag!==null&&r.tag!=="?"&&(r.dump="!<"+r.tag+"> "+r.dump)}return!0}function fpe(r,e){var t=[],i=[],n,s;for(LS(r,t,i),n=0,s=i.length;n{"use strict";var gI=r2(),S2=b2();function fI(r){return function(){throw new Error("Function "+r+" is deprecated and cannot be used.")}}Fr.exports.Type=si();Fr.exports.Schema=Yl();Fr.exports.FAILSAFE_SCHEMA=sI();Fr.exports.JSON_SCHEMA=xS();Fr.exports.CORE_SCHEMA=PS();Fr.exports.DEFAULT_SAFE_SCHEMA=Sg();Fr.exports.DEFAULT_FULL_SCHEMA=Mp();Fr.exports.load=gI.load;Fr.exports.loadAll=gI.loadAll;Fr.exports.safeLoad=gI.safeLoad;Fr.exports.safeLoadAll=gI.safeLoadAll;Fr.exports.dump=S2.dump;Fr.exports.safeDump=S2.safeDump;Fr.exports.YAMLException=Qg();Fr.exports.MINIMAL_SCHEMA=sI();Fr.exports.SAFE_SCHEMA=Sg();Fr.exports.DEFAULT_SCHEMA=Mp();Fr.exports.scan=fI("scan");Fr.exports.parse=fI("parse");Fr.exports.compose=fI("compose");Fr.exports.addConstructor=fI("addConstructor")});var P2=w((GXe,x2)=>{"use strict";var ppe=v2();x2.exports=ppe});var k2=w((YXe,D2)=>{"use strict";function dpe(r,e){function t(){this.constructor=r}t.prototype=e.prototype,r.prototype=new t}function zl(r,e,t,i){this.message=r,this.expected=e,this.found=t,this.location=i,this.name="SyntaxError",typeof Error.captureStackTrace=="function"&&Error.captureStackTrace(this,zl)}dpe(zl,Error);zl.buildMessage=function(r,e){var t={literal:function(c){return'"'+n(c.text)+'"'},class:function(c){var u="",g;for(g=0;g0){for(g=1,f=1;g({[Ke]:Ce})))},H=function(R){return R},j=function(R){return R},$=Ts("correct indentation"),V=" ",W=ar(" ",!1),Z=function(R){return R.length===pA*ug},A=function(R){return R.length===(pA+1)*ug},ae=function(){return pA++,!0},ge=function(){return pA--,!0},re=function(){return sg()},O=Ts("pseudostring"),F=/^[^\r\n\t ?:,\][{}#&*!|>'"%@`\-]/,ue=Rn(["\r",` -`," "," ","?",":",",","]","[","{","}","#","&","*","!","|",">","'",'"',"%","@","`","-"],!0,!1),he=/^[^\r\n\t ,\][{}:#"']/,ke=Rn(["\r",` -`," "," ",",","]","[","{","}",":","#",'"',"'"],!0,!1),Fe=function(){return sg().replace(/^ *| *$/g,"")},Ne="--",oe=ar("--",!1),le=/^[a-zA-Z\/0-9]/,we=Rn([["a","z"],["A","Z"],"/",["0","9"]],!1,!1),fe=/^[^\r\n\t :,]/,Ae=Rn(["\r",` -`," "," ",":",","],!0,!1),qe="null",ne=ar("null",!1),Y=function(){return null},pe="true",ie=ar("true",!1),de=function(){return!0},tt="false",Pt=ar("false",!1),It=function(){return!1},Or=Ts("string"),ii='"',gi=ar('"',!1),hr=function(){return""},fi=function(R){return R},ni=function(R){return R.join("")},Ls=/^[^"\\\0-\x1F\x7F]/,pr=Rn(['"',"\\",["\0",""],"\x7F"],!0,!1),Ei='\\"',_n=ar('\\"',!1),oa=function(){return'"'},aA="\\\\",eg=ar("\\\\",!1),Zn=function(){return"\\"},AA="\\/",aa=ar("\\/",!1),up=function(){return"/"},lA="\\b",cA=ar("\\b",!1),wr=function(){return"\b"},wl="\\f",tg=ar("\\f",!1),po=function(){return"\f"},rg="\\n",gp=ar("\\n",!1),fp=function(){return` -`},vr="\\r",se=ar("\\r",!1),Co=function(){return"\r"},Dn="\\t",ig=ar("\\t",!1),Qt=function(){return" "},Bl="\\u",kn=ar("\\u",!1),$n=function(R,q,Ce,Ke){return String.fromCharCode(parseInt(`0x${R}${q}${Ce}${Ke}`))},es=/^[0-9a-fA-F]/,gt=Rn([["0","9"],["a","f"],["A","F"]],!1,!1),mo=Ts("blank space"),At=/^[ \t]/,an=Rn([" "," "],!1,!1),S=Ts("white space"),Tt=/^[ \t\n\r]/,ng=Rn([" "," ",` -`,"\r"],!1,!1),Ql=`\r -`,hp=ar(`\r -`,!1),pp=` -`,dp=ar(` -`,!1),Cp="\r",mp=ar("\r",!1),G=0,yt=0,uA=[{line:1,column:1}],ji=0,bl=[],Xe=0,Aa;if("startRule"in e){if(!(e.startRule in i))throw new Error(`Can't start parsing from rule "`+e.startRule+'".');n=i[e.startRule]}function sg(){return r.substring(yt,G)}function bE(){return An(yt,G)}function Ep(R,q){throw q=q!==void 0?q:An(yt,G),vl([Ts(R)],r.substring(yt,G),q)}function SE(R,q){throw q=q!==void 0?q:An(yt,G),og(R,q)}function ar(R,q){return{type:"literal",text:R,ignoreCase:q}}function Rn(R,q,Ce){return{type:"class",parts:R,inverted:q,ignoreCase:Ce}}function Sl(){return{type:"any"}}function Ip(){return{type:"end"}}function Ts(R){return{type:"other",description:R}}function la(R){var q=uA[R],Ce;if(q)return q;for(Ce=R-1;!uA[Ce];)Ce--;for(q=uA[Ce],q={line:q.line,column:q.column};Ceji&&(ji=G,bl=[]),bl.push(R))}function og(R,q){return new zl(R,null,null,q)}function vl(R,q,Ce){return new zl(zl.buildMessage(R,q),R,q,Ce)}function Os(){var R;return R=ag(),R}function xl(){var R,q,Ce;for(R=G,q=[],Ce=gA();Ce!==t;)q.push(Ce),Ce=gA();return q!==t&&(yt=R,q=s(q)),R=q,R}function gA(){var R,q,Ce,Ke,Re;return R=G,q=ua(),q!==t?(r.charCodeAt(G)===45?(Ce=o,G++):(Ce=t,Xe===0&&Te(a)),Ce!==t?(Ke=Rr(),Ke!==t?(Re=ca(),Re!==t?(yt=R,q=l(Re),R=q):(G=R,R=t)):(G=R,R=t)):(G=R,R=t)):(G=R,R=t),R}function ag(){var R,q,Ce;for(R=G,q=[],Ce=Ag();Ce!==t;)q.push(Ce),Ce=Ag();return q!==t&&(yt=R,q=c(q)),R=q,R}function Ag(){var R,q,Ce,Ke,Re,ze,dt,Ft,Fn;if(R=G,q=Rr(),q===t&&(q=null),q!==t){if(Ce=G,r.charCodeAt(G)===35?(Ke=u,G++):(Ke=t,Xe===0&&Te(g)),Ke!==t){if(Re=[],ze=G,dt=G,Xe++,Ft=Us(),Xe--,Ft===t?dt=void 0:(G=dt,dt=t),dt!==t?(r.length>G?(Ft=r.charAt(G),G++):(Ft=t,Xe===0&&Te(f)),Ft!==t?(dt=[dt,Ft],ze=dt):(G=ze,ze=t)):(G=ze,ze=t),ze!==t)for(;ze!==t;)Re.push(ze),ze=G,dt=G,Xe++,Ft=Us(),Xe--,Ft===t?dt=void 0:(G=dt,dt=t),dt!==t?(r.length>G?(Ft=r.charAt(G),G++):(Ft=t,Xe===0&&Te(f)),Ft!==t?(dt=[dt,Ft],ze=dt):(G=ze,ze=t)):(G=ze,ze=t);else Re=t;Re!==t?(Ke=[Ke,Re],Ce=Ke):(G=Ce,Ce=t)}else G=Ce,Ce=t;if(Ce===t&&(Ce=null),Ce!==t){if(Ke=[],Re=Ks(),Re!==t)for(;Re!==t;)Ke.push(Re),Re=Ks();else Ke=t;Ke!==t?(yt=R,q=h(),R=q):(G=R,R=t)}else G=R,R=t}else G=R,R=t;if(R===t&&(R=G,q=ua(),q!==t?(Ce=Pl(),Ce!==t?(Ke=Rr(),Ke===t&&(Ke=null),Ke!==t?(r.charCodeAt(G)===58?(Re=p,G++):(Re=t,Xe===0&&Te(C)),Re!==t?(ze=Rr(),ze===t&&(ze=null),ze!==t?(dt=ca(),dt!==t?(yt=R,q=y(Ce,dt),R=q):(G=R,R=t)):(G=R,R=t)):(G=R,R=t)):(G=R,R=t)):(G=R,R=t)):(G=R,R=t),R===t&&(R=G,q=ua(),q!==t?(Ce=Ms(),Ce!==t?(Ke=Rr(),Ke===t&&(Ke=null),Ke!==t?(r.charCodeAt(G)===58?(Re=p,G++):(Re=t,Xe===0&&Te(C)),Re!==t?(ze=Rr(),ze===t&&(ze=null),ze!==t?(dt=ca(),dt!==t?(yt=R,q=y(Ce,dt),R=q):(G=R,R=t)):(G=R,R=t)):(G=R,R=t)):(G=R,R=t)):(G=R,R=t)):(G=R,R=t),R===t))){if(R=G,q=ua(),q!==t)if(Ce=Ms(),Ce!==t)if(Ke=Rr(),Ke!==t)if(Re=vE(),Re!==t){if(ze=[],dt=Ks(),dt!==t)for(;dt!==t;)ze.push(dt),dt=Ks();else ze=t;ze!==t?(yt=R,q=y(Ce,Re),R=q):(G=R,R=t)}else G=R,R=t;else G=R,R=t;else G=R,R=t;else G=R,R=t;if(R===t)if(R=G,q=ua(),q!==t)if(Ce=Ms(),Ce!==t){if(Ke=[],Re=G,ze=Rr(),ze===t&&(ze=null),ze!==t?(r.charCodeAt(G)===44?(dt=B,G++):(dt=t,Xe===0&&Te(v)),dt!==t?(Ft=Rr(),Ft===t&&(Ft=null),Ft!==t?(Fn=Ms(),Fn!==t?(yt=Re,ze=D(Ce,Fn),Re=ze):(G=Re,Re=t)):(G=Re,Re=t)):(G=Re,Re=t)):(G=Re,Re=t),Re!==t)for(;Re!==t;)Ke.push(Re),Re=G,ze=Rr(),ze===t&&(ze=null),ze!==t?(r.charCodeAt(G)===44?(dt=B,G++):(dt=t,Xe===0&&Te(v)),dt!==t?(Ft=Rr(),Ft===t&&(Ft=null),Ft!==t?(Fn=Ms(),Fn!==t?(yt=Re,ze=D(Ce,Fn),Re=ze):(G=Re,Re=t)):(G=Re,Re=t)):(G=Re,Re=t)):(G=Re,Re=t);else Ke=t;Ke!==t?(Re=Rr(),Re===t&&(Re=null),Re!==t?(r.charCodeAt(G)===58?(ze=p,G++):(ze=t,Xe===0&&Te(C)),ze!==t?(dt=Rr(),dt===t&&(dt=null),dt!==t?(Ft=ca(),Ft!==t?(yt=R,q=L(Ce,Ke,Ft),R=q):(G=R,R=t)):(G=R,R=t)):(G=R,R=t)):(G=R,R=t)):(G=R,R=t)}else G=R,R=t;else G=R,R=t}return R}function ca(){var R,q,Ce,Ke,Re,ze,dt;if(R=G,q=G,Xe++,Ce=G,Ke=Us(),Ke!==t?(Re=rt(),Re!==t?(r.charCodeAt(G)===45?(ze=o,G++):(ze=t,Xe===0&&Te(a)),ze!==t?(dt=Rr(),dt!==t?(Ke=[Ke,Re,ze,dt],Ce=Ke):(G=Ce,Ce=t)):(G=Ce,Ce=t)):(G=Ce,Ce=t)):(G=Ce,Ce=t),Xe--,Ce!==t?(G=q,q=void 0):q=t,q!==t?(Ce=Ks(),Ce!==t?(Ke=Eo(),Ke!==t?(Re=xl(),Re!==t?(ze=fA(),ze!==t?(yt=R,q=H(Re),R=q):(G=R,R=t)):(G=R,R=t)):(G=R,R=t)):(G=R,R=t)):(G=R,R=t),R===t&&(R=G,q=Us(),q!==t?(Ce=Eo(),Ce!==t?(Ke=ag(),Ke!==t?(Re=fA(),Re!==t?(yt=R,q=H(Ke),R=q):(G=R,R=t)):(G=R,R=t)):(G=R,R=t)):(G=R,R=t),R===t))if(R=G,q=Dl(),q!==t){if(Ce=[],Ke=Ks(),Ke!==t)for(;Ke!==t;)Ce.push(Ke),Ke=Ks();else Ce=t;Ce!==t?(yt=R,q=j(q),R=q):(G=R,R=t)}else G=R,R=t;return R}function ua(){var R,q,Ce;for(Xe++,R=G,q=[],r.charCodeAt(G)===32?(Ce=V,G++):(Ce=t,Xe===0&&Te(W));Ce!==t;)q.push(Ce),r.charCodeAt(G)===32?(Ce=V,G++):(Ce=t,Xe===0&&Te(W));return q!==t?(yt=G,Ce=Z(q),Ce?Ce=void 0:Ce=t,Ce!==t?(q=[q,Ce],R=q):(G=R,R=t)):(G=R,R=t),Xe--,R===t&&(q=t,Xe===0&&Te($)),R}function rt(){var R,q,Ce;for(R=G,q=[],r.charCodeAt(G)===32?(Ce=V,G++):(Ce=t,Xe===0&&Te(W));Ce!==t;)q.push(Ce),r.charCodeAt(G)===32?(Ce=V,G++):(Ce=t,Xe===0&&Te(W));return q!==t?(yt=G,Ce=A(q),Ce?Ce=void 0:Ce=t,Ce!==t?(q=[q,Ce],R=q):(G=R,R=t)):(G=R,R=t),R}function Eo(){var R;return yt=G,R=ae(),R?R=void 0:R=t,R}function fA(){var R;return yt=G,R=ge(),R?R=void 0:R=t,R}function Pl(){var R;return R=kl(),R===t&&(R=yp()),R}function Ms(){var R,q,Ce;if(R=kl(),R===t){if(R=G,q=[],Ce=lg(),Ce!==t)for(;Ce!==t;)q.push(Ce),Ce=lg();else q=t;q!==t&&(yt=R,q=re()),R=q}return R}function Dl(){var R;return R=wp(),R===t&&(R=xE(),R===t&&(R=kl(),R===t&&(R=yp()))),R}function vE(){var R;return R=wp(),R===t&&(R=kl(),R===t&&(R=lg())),R}function yp(){var R,q,Ce,Ke,Re,ze;if(Xe++,R=G,F.test(r.charAt(G))?(q=r.charAt(G),G++):(q=t,Xe===0&&Te(ue)),q!==t){for(Ce=[],Ke=G,Re=Rr(),Re===t&&(Re=null),Re!==t?(he.test(r.charAt(G))?(ze=r.charAt(G),G++):(ze=t,Xe===0&&Te(ke)),ze!==t?(Re=[Re,ze],Ke=Re):(G=Ke,Ke=t)):(G=Ke,Ke=t);Ke!==t;)Ce.push(Ke),Ke=G,Re=Rr(),Re===t&&(Re=null),Re!==t?(he.test(r.charAt(G))?(ze=r.charAt(G),G++):(ze=t,Xe===0&&Te(ke)),ze!==t?(Re=[Re,ze],Ke=Re):(G=Ke,Ke=t)):(G=Ke,Ke=t);Ce!==t?(yt=R,q=Fe(),R=q):(G=R,R=t)}else G=R,R=t;return Xe--,R===t&&(q=t,Xe===0&&Te(O)),R}function lg(){var R,q,Ce,Ke,Re;if(R=G,r.substr(G,2)===Ne?(q=Ne,G+=2):(q=t,Xe===0&&Te(oe)),q===t&&(q=null),q!==t)if(le.test(r.charAt(G))?(Ce=r.charAt(G),G++):(Ce=t,Xe===0&&Te(we)),Ce!==t){for(Ke=[],fe.test(r.charAt(G))?(Re=r.charAt(G),G++):(Re=t,Xe===0&&Te(Ae));Re!==t;)Ke.push(Re),fe.test(r.charAt(G))?(Re=r.charAt(G),G++):(Re=t,Xe===0&&Te(Ae));Ke!==t?(yt=R,q=Fe(),R=q):(G=R,R=t)}else G=R,R=t;else G=R,R=t;return R}function wp(){var R,q;return R=G,r.substr(G,4)===qe?(q=qe,G+=4):(q=t,Xe===0&&Te(ne)),q!==t&&(yt=R,q=Y()),R=q,R}function xE(){var R,q;return R=G,r.substr(G,4)===pe?(q=pe,G+=4):(q=t,Xe===0&&Te(ie)),q!==t&&(yt=R,q=de()),R=q,R===t&&(R=G,r.substr(G,5)===tt?(q=tt,G+=5):(q=t,Xe===0&&Te(Pt)),q!==t&&(yt=R,q=It()),R=q),R}function kl(){var R,q,Ce,Ke;return Xe++,R=G,r.charCodeAt(G)===34?(q=ii,G++):(q=t,Xe===0&&Te(gi)),q!==t?(r.charCodeAt(G)===34?(Ce=ii,G++):(Ce=t,Xe===0&&Te(gi)),Ce!==t?(yt=R,q=hr(),R=q):(G=R,R=t)):(G=R,R=t),R===t&&(R=G,r.charCodeAt(G)===34?(q=ii,G++):(q=t,Xe===0&&Te(gi)),q!==t?(Ce=PE(),Ce!==t?(r.charCodeAt(G)===34?(Ke=ii,G++):(Ke=t,Xe===0&&Te(gi)),Ke!==t?(yt=R,q=fi(Ce),R=q):(G=R,R=t)):(G=R,R=t)):(G=R,R=t)),Xe--,R===t&&(q=t,Xe===0&&Te(Or)),R}function PE(){var R,q,Ce;if(R=G,q=[],Ce=cg(),Ce!==t)for(;Ce!==t;)q.push(Ce),Ce=cg();else q=t;return q!==t&&(yt=R,q=ni(q)),R=q,R}function cg(){var R,q,Ce,Ke,Re,ze;return Ls.test(r.charAt(G))?(R=r.charAt(G),G++):(R=t,Xe===0&&Te(pr)),R===t&&(R=G,r.substr(G,2)===Ei?(q=Ei,G+=2):(q=t,Xe===0&&Te(_n)),q!==t&&(yt=R,q=oa()),R=q,R===t&&(R=G,r.substr(G,2)===aA?(q=aA,G+=2):(q=t,Xe===0&&Te(eg)),q!==t&&(yt=R,q=Zn()),R=q,R===t&&(R=G,r.substr(G,2)===AA?(q=AA,G+=2):(q=t,Xe===0&&Te(aa)),q!==t&&(yt=R,q=up()),R=q,R===t&&(R=G,r.substr(G,2)===lA?(q=lA,G+=2):(q=t,Xe===0&&Te(cA)),q!==t&&(yt=R,q=wr()),R=q,R===t&&(R=G,r.substr(G,2)===wl?(q=wl,G+=2):(q=t,Xe===0&&Te(tg)),q!==t&&(yt=R,q=po()),R=q,R===t&&(R=G,r.substr(G,2)===rg?(q=rg,G+=2):(q=t,Xe===0&&Te(gp)),q!==t&&(yt=R,q=fp()),R=q,R===t&&(R=G,r.substr(G,2)===vr?(q=vr,G+=2):(q=t,Xe===0&&Te(se)),q!==t&&(yt=R,q=Co()),R=q,R===t&&(R=G,r.substr(G,2)===Dn?(q=Dn,G+=2):(q=t,Xe===0&&Te(ig)),q!==t&&(yt=R,q=Qt()),R=q,R===t&&(R=G,r.substr(G,2)===Bl?(q=Bl,G+=2):(q=t,Xe===0&&Te(kn)),q!==t?(Ce=hA(),Ce!==t?(Ke=hA(),Ke!==t?(Re=hA(),Re!==t?(ze=hA(),ze!==t?(yt=R,q=$n(Ce,Ke,Re,ze),R=q):(G=R,R=t)):(G=R,R=t)):(G=R,R=t)):(G=R,R=t)):(G=R,R=t)))))))))),R}function hA(){var R;return es.test(r.charAt(G))?(R=r.charAt(G),G++):(R=t,Xe===0&&Te(gt)),R}function Rr(){var R,q;if(Xe++,R=[],At.test(r.charAt(G))?(q=r.charAt(G),G++):(q=t,Xe===0&&Te(an)),q!==t)for(;q!==t;)R.push(q),At.test(r.charAt(G))?(q=r.charAt(G),G++):(q=t,Xe===0&&Te(an));else R=t;return Xe--,R===t&&(q=t,Xe===0&&Te(mo)),R}function DE(){var R,q;if(Xe++,R=[],Tt.test(r.charAt(G))?(q=r.charAt(G),G++):(q=t,Xe===0&&Te(ng)),q!==t)for(;q!==t;)R.push(q),Tt.test(r.charAt(G))?(q=r.charAt(G),G++):(q=t,Xe===0&&Te(ng));else R=t;return Xe--,R===t&&(q=t,Xe===0&&Te(S)),R}function Ks(){var R,q,Ce,Ke,Re,ze;if(R=G,q=Us(),q!==t){for(Ce=[],Ke=G,Re=Rr(),Re===t&&(Re=null),Re!==t?(ze=Us(),ze!==t?(Re=[Re,ze],Ke=Re):(G=Ke,Ke=t)):(G=Ke,Ke=t);Ke!==t;)Ce.push(Ke),Ke=G,Re=Rr(),Re===t&&(Re=null),Re!==t?(ze=Us(),ze!==t?(Re=[Re,ze],Ke=Re):(G=Ke,Ke=t)):(G=Ke,Ke=t);Ce!==t?(q=[q,Ce],R=q):(G=R,R=t)}else G=R,R=t;return R}function Us(){var R;return r.substr(G,2)===Ql?(R=Ql,G+=2):(R=t,Xe===0&&Te(hp)),R===t&&(r.charCodeAt(G)===10?(R=pp,G++):(R=t,Xe===0&&Te(dp)),R===t&&(r.charCodeAt(G)===13?(R=Cp,G++):(R=t,Xe===0&&Te(mp)))),R}let ug=2,pA=0;if(Aa=n(),Aa!==t&&G===r.length)return Aa;throw Aa!==t&&G{"use strict";var wpe=r=>{let e=!1,t=!1,i=!1;for(let n=0;n{if(!(typeof r=="string"||Array.isArray(r)))throw new TypeError("Expected the input to be `string | string[]`");e=Object.assign({pascalCase:!1},e);let t=n=>e.pascalCase?n.charAt(0).toUpperCase()+n.slice(1):n;return Array.isArray(r)?r=r.map(n=>n.trim()).filter(n=>n.length).join("-"):r=r.trim(),r.length===0?"":r.length===1?e.pascalCase?r.toUpperCase():r.toLowerCase():(r!==r.toLowerCase()&&(r=wpe(r)),r=r.replace(/^[_.\- ]+/,"").toLowerCase().replace(/[_.\- ]+(\w|$)/g,(n,s)=>s.toUpperCase()).replace(/\d+(\w|$)/g,n=>n.toUpperCase()),t(r))};KS.exports=T2;KS.exports.default=T2});var M2=w((VXe,Bpe)=>{Bpe.exports=[{name:"AppVeyor",constant:"APPVEYOR",env:"APPVEYOR",pr:"APPVEYOR_PULL_REQUEST_NUMBER"},{name:"Azure Pipelines",constant:"AZURE_PIPELINES",env:"SYSTEM_TEAMFOUNDATIONCOLLECTIONURI",pr:"SYSTEM_PULLREQUEST_PULLREQUESTID"},{name:"Appcircle",constant:"APPCIRCLE",env:"AC_APPCIRCLE"},{name:"Bamboo",constant:"BAMBOO",env:"bamboo_planKey"},{name:"Bitbucket Pipelines",constant:"BITBUCKET",env:"BITBUCKET_COMMIT",pr:"BITBUCKET_PR_ID"},{name:"Bitrise",constant:"BITRISE",env:"BITRISE_IO",pr:"BITRISE_PULL_REQUEST"},{name:"Buddy",constant:"BUDDY",env:"BUDDY_WORKSPACE_ID",pr:"BUDDY_EXECUTION_PULL_REQUEST_ID"},{name:"Buildkite",constant:"BUILDKITE",env:"BUILDKITE",pr:{env:"BUILDKITE_PULL_REQUEST",ne:"false"}},{name:"CircleCI",constant:"CIRCLE",env:"CIRCLECI",pr:"CIRCLE_PULL_REQUEST"},{name:"Cirrus CI",constant:"CIRRUS",env:"CIRRUS_CI",pr:"CIRRUS_PR"},{name:"AWS CodeBuild",constant:"CODEBUILD",env:"CODEBUILD_BUILD_ARN"},{name:"Codefresh",constant:"CODEFRESH",env:"CF_BUILD_ID",pr:{any:["CF_PULL_REQUEST_NUMBER","CF_PULL_REQUEST_ID"]}},{name:"Codeship",constant:"CODESHIP",env:{CI_NAME:"codeship"}},{name:"Drone",constant:"DRONE",env:"DRONE",pr:{DRONE_BUILD_EVENT:"pull_request"}},{name:"dsari",constant:"DSARI",env:"DSARI"},{name:"GitHub Actions",constant:"GITHUB_ACTIONS",env:"GITHUB_ACTIONS",pr:{GITHUB_EVENT_NAME:"pull_request"}},{name:"GitLab CI",constant:"GITLAB",env:"GITLAB_CI",pr:"CI_MERGE_REQUEST_ID"},{name:"GoCD",constant:"GOCD",env:"GO_PIPELINE_LABEL"},{name:"LayerCI",constant:"LAYERCI",env:"LAYERCI",pr:"LAYERCI_PULL_REQUEST"},{name:"Hudson",constant:"HUDSON",env:"HUDSON_URL"},{name:"Jenkins",constant:"JENKINS",env:["JENKINS_URL","BUILD_ID"],pr:{any:["ghprbPullId","CHANGE_ID"]}},{name:"Magnum CI",constant:"MAGNUM",env:"MAGNUM"},{name:"Netlify CI",constant:"NETLIFY",env:"NETLIFY",pr:{env:"PULL_REQUEST",ne:"false"}},{name:"Nevercode",constant:"NEVERCODE",env:"NEVERCODE",pr:{env:"NEVERCODE_PULL_REQUEST",ne:"false"}},{name:"Render",constant:"RENDER",env:"RENDER",pr:{IS_PULL_REQUEST:"true"}},{name:"Sail CI",constant:"SAIL",env:"SAILCI",pr:"SAIL_PULL_REQUEST_NUMBER"},{name:"Semaphore",constant:"SEMAPHORE",env:"SEMAPHORE",pr:"PULL_REQUEST_NUMBER"},{name:"Screwdriver",constant:"SCREWDRIVER",env:"SCREWDRIVER",pr:{env:"SD_PULL_REQUEST",ne:"false"}},{name:"Shippable",constant:"SHIPPABLE",env:"SHIPPABLE",pr:{IS_PULL_REQUEST:"true"}},{name:"Solano CI",constant:"SOLANO",env:"TDDIUM",pr:"TDDIUM_PR_ID"},{name:"Strider CD",constant:"STRIDER",env:"STRIDER"},{name:"TaskCluster",constant:"TASKCLUSTER",env:["TASK_ID","RUN_ID"]},{name:"TeamCity",constant:"TEAMCITY",env:"TEAMCITY_VERSION"},{name:"Travis CI",constant:"TRAVIS",env:"TRAVIS",pr:{env:"TRAVIS_PULL_REQUEST",ne:"false"}},{name:"Vercel",constant:"VERCEL",env:"NOW_BUILDER"},{name:"Visual Studio App Center",constant:"APPCENTER",env:"APPCENTER_BUILD_ID"}]});var Vl=w(On=>{"use strict";var U2=M2(),Qo=process.env;Object.defineProperty(On,"_vendors",{value:U2.map(function(r){return r.constant})});On.name=null;On.isPR=null;U2.forEach(function(r){let t=(Array.isArray(r.env)?r.env:[r.env]).every(function(i){return K2(i)});if(On[r.constant]=t,t)switch(On.name=r.name,typeof r.pr){case"string":On.isPR=!!Qo[r.pr];break;case"object":"env"in r.pr?On.isPR=r.pr.env in Qo&&Qo[r.pr.env]!==r.pr.ne:"any"in r.pr?On.isPR=r.pr.any.some(function(i){return!!Qo[i]}):On.isPR=K2(r.pr);break;default:On.isPR=null}});On.isCI=!!(Qo.CI||Qo.CONTINUOUS_INTEGRATION||Qo.BUILD_NUMBER||Qo.RUN_ID||On.name);function K2(r){return typeof r=="string"?!!Qo[r]:Object.keys(r).every(function(e){return Qo[e]===r[e]})}});var gn={};ut(gn,{KeyRelationship:()=>Xl,applyCascade:()=>zp,base64RegExp:()=>q2,colorStringAlphaRegExp:()=>j2,colorStringRegExp:()=>Y2,computeKey:()=>BA,getPrintable:()=>Vr,hasExactLength:()=>X2,hasForbiddenKeys:()=>tde,hasKeyRelationship:()=>JS,hasMaxLength:()=>Mpe,hasMinLength:()=>Ope,hasMutuallyExclusiveKeys:()=>rde,hasRequiredKeys:()=>ede,hasUniqueItems:()=>Kpe,isArray:()=>Ppe,isAtLeast:()=>Gpe,isAtMost:()=>Ype,isBase64:()=>Zpe,isBoolean:()=>Spe,isDate:()=>xpe,isDict:()=>kpe,isEnum:()=>Wi,isHexColor:()=>_pe,isISO8601:()=>Xpe,isInExclusiveRange:()=>qpe,isInInclusiveRange:()=>jpe,isInstanceOf:()=>Fpe,isInteger:()=>Jpe,isJSON:()=>$pe,isLiteral:()=>Qpe,isLowerCase:()=>Wpe,isNegative:()=>Upe,isNullable:()=>Tpe,isNumber:()=>vpe,isObject:()=>Rpe,isOneOf:()=>Npe,isOptional:()=>Lpe,isPositive:()=>Hpe,isString:()=>Wp,isTuple:()=>Dpe,isUUID4:()=>Vpe,isUnknown:()=>V2,isUpperCase:()=>zpe,iso8601RegExp:()=>qS,makeCoercionFn:()=>_l,makeSetter:()=>z2,makeTrait:()=>W2,makeValidator:()=>bt,matchesRegExp:()=>Vp,plural:()=>EI,pushError:()=>pt,simpleKeyRegExp:()=>G2,uuid4RegExp:()=>J2});function bt({test:r}){return W2(r)()}function Vr(r){return r===null?"null":r===void 0?"undefined":r===""?"an empty string":JSON.stringify(r)}function BA(r,e){var t,i,n;return typeof e=="number"?`${(t=r==null?void 0:r.p)!==null&&t!==void 0?t:"."}[${e}]`:G2.test(e)?`${(i=r==null?void 0:r.p)!==null&&i!==void 0?i:""}.${e}`:`${(n=r==null?void 0:r.p)!==null&&n!==void 0?n:"."}[${JSON.stringify(e)}]`}function _l(r,e){return t=>{let i=r[e];return r[e]=t,_l(r,e).bind(null,i)}}function z2(r,e){return t=>{r[e]=t}}function EI(r,e,t){return r===1?e:t}function pt({errors:r,p:e}={},t){return r==null||r.push(`${e!=null?e:"."}: ${t}`),!1}function Qpe(r){return bt({test:(e,t)=>e!==r?pt(t,`Expected a literal (got ${Vr(r)})`):!0})}function Wi(r){let e=Array.isArray(r)?r:Object.values(r),t=new Set(e);return bt({test:(i,n)=>t.has(i)?!0:pt(n,`Expected a valid enumeration value (got ${Vr(i)})`)})}var G2,Y2,j2,q2,J2,qS,W2,V2,Wp,bpe,Spe,vpe,xpe,Ppe,Dpe,kpe,Rpe,Fpe,Npe,zp,Lpe,Tpe,Ope,Mpe,X2,Kpe,Upe,Hpe,Gpe,Ype,jpe,qpe,Jpe,Vp,Wpe,zpe,Vpe,Xpe,_pe,Zpe,$pe,ede,tde,rde,Xl,ide,JS,ns=Yue(()=>{G2=/^[a-zA-Z_][a-zA-Z0-9_]*$/,Y2=/^#[0-9a-f]{6}$/i,j2=/^#[0-9a-f]{6}([0-9a-f]{2})?$/i,q2=/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/,J2=/^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89aAbB][a-f0-9]{3}-[a-f0-9]{12}$/i,qS=/^(?:[1-9]\d{3}(-?)(?:(?:0[1-9]|1[0-2])\1(?:0[1-9]|1\d|2[0-8])|(?:0[13-9]|1[0-2])\1(?:29|30)|(?:0[13578]|1[02])(?:\1)31|00[1-9]|0[1-9]\d|[12]\d{2}|3(?:[0-5]\d|6[0-5]))|(?:[1-9]\d(?:0[48]|[2468][048]|[13579][26])|(?:[2468][048]|[13579][26])00)(?:(-?)02(?:\2)29|-?366))T(?:[01]\d|2[0-3])(:?)[0-5]\d(?:\3[0-5]\d)?(?:Z|[+-][01]\d(?:\3[0-5]\d)?)$/,W2=r=>()=>r;V2=()=>bt({test:(r,e)=>!0});Wp=()=>bt({test:(r,e)=>typeof r!="string"?pt(e,`Expected a string (got ${Vr(r)})`):!0});bpe=new Map([["true",!0],["True",!0],["1",!0],[1,!0],["false",!1],["False",!1],["0",!1],[0,!1]]),Spe=()=>bt({test:(r,e)=>{var t;if(typeof r!="boolean"){if(typeof(e==null?void 0:e.coercions)<"u"){if(typeof(e==null?void 0:e.coercion)>"u")return pt(e,"Unbound coercion result");let i=bpe.get(r);if(typeof i<"u")return e.coercions.push([(t=e.p)!==null&&t!==void 0?t:".",e.coercion.bind(null,i)]),!0}return pt(e,`Expected a boolean (got ${Vr(r)})`)}return!0}}),vpe=()=>bt({test:(r,e)=>{var t;if(typeof r!="number"){if(typeof(e==null?void 0:e.coercions)<"u"){if(typeof(e==null?void 0:e.coercion)>"u")return pt(e,"Unbound coercion result");let i;if(typeof r=="string"){let n;try{n=JSON.parse(r)}catch{}if(typeof n=="number")if(JSON.stringify(n)===r)i=n;else return pt(e,`Received a number that can't be safely represented by the runtime (${r})`)}if(typeof i<"u")return e.coercions.push([(t=e.p)!==null&&t!==void 0?t:".",e.coercion.bind(null,i)]),!0}return pt(e,`Expected a number (got ${Vr(r)})`)}return!0}}),xpe=()=>bt({test:(r,e)=>{var t;if(!(r instanceof Date)){if(typeof(e==null?void 0:e.coercions)<"u"){if(typeof(e==null?void 0:e.coercion)>"u")return pt(e,"Unbound coercion result");let i;if(typeof r=="string"&&qS.test(r))i=new Date(r);else{let n;if(typeof r=="string"){let s;try{s=JSON.parse(r)}catch{}typeof s=="number"&&(n=s)}else typeof r=="number"&&(n=r);if(typeof n<"u")if(Number.isSafeInteger(n)||!Number.isSafeInteger(n*1e3))i=new Date(n*1e3);else return pt(e,`Received a timestamp that can't be safely represented by the runtime (${r})`)}if(typeof i<"u")return e.coercions.push([(t=e.p)!==null&&t!==void 0?t:".",e.coercion.bind(null,i)]),!0}return pt(e,`Expected a date (got ${Vr(r)})`)}return!0}}),Ppe=(r,{delimiter:e}={})=>bt({test:(t,i)=>{var n;if(typeof t=="string"&&typeof e<"u"&&typeof(i==null?void 0:i.coercions)<"u"){if(typeof(i==null?void 0:i.coercion)>"u")return pt(i,"Unbound coercion result");t=t.split(e),i.coercions.push([(n=i.p)!==null&&n!==void 0?n:".",i.coercion.bind(null,t)])}if(!Array.isArray(t))return pt(i,`Expected an array (got ${Vr(t)})`);let s=!0;for(let o=0,a=t.length;o{let t=X2(r.length);return bt({test:(i,n)=>{var s;if(typeof i=="string"&&typeof e<"u"&&typeof(n==null?void 0:n.coercions)<"u"){if(typeof(n==null?void 0:n.coercion)>"u")return pt(n,"Unbound coercion result");i=i.split(e),n.coercions.push([(s=n.p)!==null&&s!==void 0?s:".",n.coercion.bind(null,i)])}if(!Array.isArray(i))return pt(n,`Expected a tuple (got ${Vr(i)})`);let o=t(i,Object.assign({},n));for(let a=0,l=i.length;abt({test:(t,i)=>{if(typeof t!="object"||t===null)return pt(i,`Expected an object (got ${Vr(t)})`);let n=Object.keys(t),s=!0;for(let o=0,a=n.length;o{let t=Object.keys(r);return bt({test:(i,n)=>{if(typeof i!="object"||i===null)return pt(n,`Expected an object (got ${Vr(i)})`);let s=new Set([...t,...Object.keys(i)]),o={},a=!0;for(let l of s){if(l==="constructor"||l==="__proto__")a=pt(Object.assign(Object.assign({},n),{p:BA(n,l)}),"Unsafe property name");else{let c=Object.prototype.hasOwnProperty.call(r,l)?r[l]:void 0,u=Object.prototype.hasOwnProperty.call(i,l)?i[l]:void 0;typeof c<"u"?a=c(u,Object.assign(Object.assign({},n),{p:BA(n,l),coercion:_l(i,l)}))&&a:e===null?a=pt(Object.assign(Object.assign({},n),{p:BA(n,l)}),`Extraneous property (got ${Vr(u)})`):Object.defineProperty(o,l,{enumerable:!0,get:()=>u,set:z2(i,l)})}if(!a&&(n==null?void 0:n.errors)==null)break}return e!==null&&(a||(n==null?void 0:n.errors)!=null)&&(a=e(o,n)&&a),a}})},Fpe=r=>bt({test:(e,t)=>e instanceof r?!0:pt(t,`Expected an instance of ${r.name} (got ${Vr(e)})`)}),Npe=(r,{exclusive:e=!1}={})=>bt({test:(t,i)=>{var n,s,o;let a=[],l=typeof(i==null?void 0:i.errors)<"u"?[]:void 0;for(let c=0,u=r.length;c1?pt(i,`Expected to match exactly a single predicate (matched ${a.join(", ")})`):(o=i==null?void 0:i.errors)===null||o===void 0||o.push(...l),!1}}),zp=(r,e)=>bt({test:(t,i)=>{var n,s;let o={value:t},a=typeof(i==null?void 0:i.coercions)<"u"?_l(o,"value"):void 0,l=typeof(i==null?void 0:i.coercions)<"u"?[]:void 0;if(!r(t,Object.assign(Object.assign({},i),{coercion:a,coercions:l})))return!1;let c=[];if(typeof l<"u")for(let[,u]of l)c.push(u());try{if(typeof(i==null?void 0:i.coercions)<"u"){if(o.value!==t){if(typeof(i==null?void 0:i.coercion)>"u")return pt(i,"Unbound coercion result");i.coercions.push([(n=i.p)!==null&&n!==void 0?n:".",i.coercion.bind(null,o.value)])}(s=i==null?void 0:i.coercions)===null||s===void 0||s.push(...l)}return e.every(u=>u(o.value,i))}finally{for(let u of c)u()}}}),Lpe=r=>bt({test:(e,t)=>typeof e>"u"?!0:r(e,t)}),Tpe=r=>bt({test:(e,t)=>e===null?!0:r(e,t)}),Ope=r=>bt({test:(e,t)=>e.length>=r?!0:pt(t,`Expected to have a length of at least ${r} elements (got ${e.length})`)}),Mpe=r=>bt({test:(e,t)=>e.length<=r?!0:pt(t,`Expected to have a length of at most ${r} elements (got ${e.length})`)}),X2=r=>bt({test:(e,t)=>e.length!==r?pt(t,`Expected to have a length of exactly ${r} elements (got ${e.length})`):!0}),Kpe=({map:r}={})=>bt({test:(e,t)=>{let i=new Set,n=new Set;for(let s=0,o=e.length;sbt({test:(r,e)=>r<=0?!0:pt(e,`Expected to be negative (got ${r})`)}),Hpe=()=>bt({test:(r,e)=>r>=0?!0:pt(e,`Expected to be positive (got ${r})`)}),Gpe=r=>bt({test:(e,t)=>e>=r?!0:pt(t,`Expected to be at least ${r} (got ${e})`)}),Ype=r=>bt({test:(e,t)=>e<=r?!0:pt(t,`Expected to be at most ${r} (got ${e})`)}),jpe=(r,e)=>bt({test:(t,i)=>t>=r&&t<=e?!0:pt(i,`Expected to be in the [${r}; ${e}] range (got ${t})`)}),qpe=(r,e)=>bt({test:(t,i)=>t>=r&&tbt({test:(e,t)=>e!==Math.round(e)?pt(t,`Expected to be an integer (got ${e})`):Number.isSafeInteger(e)?!0:pt(t,`Expected to be a safe integer (got ${e})`)}),Vp=r=>bt({test:(e,t)=>r.test(e)?!0:pt(t,`Expected to match the pattern ${r.toString()} (got ${Vr(e)})`)}),Wpe=()=>bt({test:(r,e)=>r!==r.toLowerCase()?pt(e,`Expected to be all-lowercase (got ${r})`):!0}),zpe=()=>bt({test:(r,e)=>r!==r.toUpperCase()?pt(e,`Expected to be all-uppercase (got ${r})`):!0}),Vpe=()=>bt({test:(r,e)=>J2.test(r)?!0:pt(e,`Expected to be a valid UUID v4 (got ${Vr(r)})`)}),Xpe=()=>bt({test:(r,e)=>qS.test(r)?!1:pt(e,`Expected to be a valid ISO 8601 date string (got ${Vr(r)})`)}),_pe=({alpha:r=!1})=>bt({test:(e,t)=>(r?Y2.test(e):j2.test(e))?!0:pt(t,`Expected to be a valid hexadecimal color string (got ${Vr(e)})`)}),Zpe=()=>bt({test:(r,e)=>q2.test(r)?!0:pt(e,`Expected to be a valid base 64 string (got ${Vr(r)})`)}),$pe=(r=V2())=>bt({test:(e,t)=>{let i;try{i=JSON.parse(e)}catch{return pt(t,`Expected to be a valid JSON string (got ${Vr(e)})`)}return r(i,t)}}),ede=r=>{let e=new Set(r);return bt({test:(t,i)=>{let n=new Set(Object.keys(t)),s=[];for(let o of e)n.has(o)||s.push(o);return s.length>0?pt(i,`Missing required ${EI(s.length,"property","properties")} ${s.map(o=>`"${o}"`).join(", ")}`):!0}})},tde=r=>{let e=new Set(r);return bt({test:(t,i)=>{let n=new Set(Object.keys(t)),s=[];for(let o of e)n.has(o)&&s.push(o);return s.length>0?pt(i,`Forbidden ${EI(s.length,"property","properties")} ${s.map(o=>`"${o}"`).join(", ")}`):!0}})},rde=r=>{let e=new Set(r);return bt({test:(t,i)=>{let n=new Set(Object.keys(t)),s=[];for(let o of e)n.has(o)&&s.push(o);return s.length>1?pt(i,`Mutually exclusive properties ${s.map(o=>`"${o}"`).join(", ")}`):!0}})};(function(r){r.Forbids="Forbids",r.Requires="Requires"})(Xl||(Xl={}));ide={[Xl.Forbids]:{expect:!1,message:"forbids using"},[Xl.Requires]:{expect:!0,message:"requires using"}},JS=(r,e,t,{ignore:i=[]}={})=>{let n=new Set(i),s=new Set(t),o=ide[e];return bt({test:(a,l)=>{let c=new Set(Object.keys(a));if(!c.has(r)||n.has(a[r]))return!0;let u=[];for(let g of s)(c.has(g)&&!n.has(a[g]))!==o.expect&&u.push(g);return u.length>=1?pt(l,`Property "${r}" ${o.message} ${EI(u.length,"property","properties")} ${u.map(g=>`"${g}"`).join(", ")}`):!0}})}});var fH=w((V_e,gH)=>{"use strict";gH.exports=(r,...e)=>new Promise(t=>{t(r(...e))})});var Tg=w((X_e,ev)=>{"use strict";var Ide=fH(),hH=r=>{if(r<1)throw new TypeError("Expected `concurrency` to be a number from 1 and up");let e=[],t=0,i=()=>{t--,e.length>0&&e.shift()()},n=(a,l,...c)=>{t++;let u=Ide(a,...c);l(u),u.then(i,i)},s=(a,l,...c)=>{tnew Promise(c=>s(a,c,...l));return Object.defineProperties(o,{activeCount:{get:()=>t},pendingCount:{get:()=>e.length}}),o};ev.exports=hH;ev.exports.default=hH});var ed=w((Z_e,pH)=>{var yde="2.0.0",wde=Number.MAX_SAFE_INTEGER||9007199254740991,Bde=16;pH.exports={SEMVER_SPEC_VERSION:yde,MAX_LENGTH:256,MAX_SAFE_INTEGER:wde,MAX_SAFE_COMPONENT_LENGTH:Bde}});var td=w(($_e,dH)=>{var Qde=typeof process=="object"&&process.env&&process.env.NODE_DEBUG&&/\bsemver\b/i.test(process.env.NODE_DEBUG)?(...r)=>console.error("SEMVER",...r):()=>{};dH.exports=Qde});var Zl=w((bA,CH)=>{var{MAX_SAFE_COMPONENT_LENGTH:tv}=ed(),bde=td();bA=CH.exports={};var Sde=bA.re=[],$e=bA.src=[],et=bA.t={},vde=0,St=(r,e,t)=>{let i=vde++;bde(i,e),et[r]=i,$e[i]=e,Sde[i]=new RegExp(e,t?"g":void 0)};St("NUMERICIDENTIFIER","0|[1-9]\\d*");St("NUMERICIDENTIFIERLOOSE","[0-9]+");St("NONNUMERICIDENTIFIER","\\d*[a-zA-Z-][a-zA-Z0-9-]*");St("MAINVERSION",`(${$e[et.NUMERICIDENTIFIER]})\\.(${$e[et.NUMERICIDENTIFIER]})\\.(${$e[et.NUMERICIDENTIFIER]})`);St("MAINVERSIONLOOSE",`(${$e[et.NUMERICIDENTIFIERLOOSE]})\\.(${$e[et.NUMERICIDENTIFIERLOOSE]})\\.(${$e[et.NUMERICIDENTIFIERLOOSE]})`);St("PRERELEASEIDENTIFIER",`(?:${$e[et.NUMERICIDENTIFIER]}|${$e[et.NONNUMERICIDENTIFIER]})`);St("PRERELEASEIDENTIFIERLOOSE",`(?:${$e[et.NUMERICIDENTIFIERLOOSE]}|${$e[et.NONNUMERICIDENTIFIER]})`);St("PRERELEASE",`(?:-(${$e[et.PRERELEASEIDENTIFIER]}(?:\\.${$e[et.PRERELEASEIDENTIFIER]})*))`);St("PRERELEASELOOSE",`(?:-?(${$e[et.PRERELEASEIDENTIFIERLOOSE]}(?:\\.${$e[et.PRERELEASEIDENTIFIERLOOSE]})*))`);St("BUILDIDENTIFIER","[0-9A-Za-z-]+");St("BUILD",`(?:\\+(${$e[et.BUILDIDENTIFIER]}(?:\\.${$e[et.BUILDIDENTIFIER]})*))`);St("FULLPLAIN",`v?${$e[et.MAINVERSION]}${$e[et.PRERELEASE]}?${$e[et.BUILD]}?`);St("FULL",`^${$e[et.FULLPLAIN]}$`);St("LOOSEPLAIN",`[v=\\s]*${$e[et.MAINVERSIONLOOSE]}${$e[et.PRERELEASELOOSE]}?${$e[et.BUILD]}?`);St("LOOSE",`^${$e[et.LOOSEPLAIN]}$`);St("GTLT","((?:<|>)?=?)");St("XRANGEIDENTIFIERLOOSE",`${$e[et.NUMERICIDENTIFIERLOOSE]}|x|X|\\*`);St("XRANGEIDENTIFIER",`${$e[et.NUMERICIDENTIFIER]}|x|X|\\*`);St("XRANGEPLAIN",`[v=\\s]*(${$e[et.XRANGEIDENTIFIER]})(?:\\.(${$e[et.XRANGEIDENTIFIER]})(?:\\.(${$e[et.XRANGEIDENTIFIER]})(?:${$e[et.PRERELEASE]})?${$e[et.BUILD]}?)?)?`);St("XRANGEPLAINLOOSE",`[v=\\s]*(${$e[et.XRANGEIDENTIFIERLOOSE]})(?:\\.(${$e[et.XRANGEIDENTIFIERLOOSE]})(?:\\.(${$e[et.XRANGEIDENTIFIERLOOSE]})(?:${$e[et.PRERELEASELOOSE]})?${$e[et.BUILD]}?)?)?`);St("XRANGE",`^${$e[et.GTLT]}\\s*${$e[et.XRANGEPLAIN]}$`);St("XRANGELOOSE",`^${$e[et.GTLT]}\\s*${$e[et.XRANGEPLAINLOOSE]}$`);St("COERCE",`(^|[^\\d])(\\d{1,${tv}})(?:\\.(\\d{1,${tv}}))?(?:\\.(\\d{1,${tv}}))?(?:$|[^\\d])`);St("COERCERTL",$e[et.COERCE],!0);St("LONETILDE","(?:~>?)");St("TILDETRIM",`(\\s*)${$e[et.LONETILDE]}\\s+`,!0);bA.tildeTrimReplace="$1~";St("TILDE",`^${$e[et.LONETILDE]}${$e[et.XRANGEPLAIN]}$`);St("TILDELOOSE",`^${$e[et.LONETILDE]}${$e[et.XRANGEPLAINLOOSE]}$`);St("LONECARET","(?:\\^)");St("CARETTRIM",`(\\s*)${$e[et.LONECARET]}\\s+`,!0);bA.caretTrimReplace="$1^";St("CARET",`^${$e[et.LONECARET]}${$e[et.XRANGEPLAIN]}$`);St("CARETLOOSE",`^${$e[et.LONECARET]}${$e[et.XRANGEPLAINLOOSE]}$`);St("COMPARATORLOOSE",`^${$e[et.GTLT]}\\s*(${$e[et.LOOSEPLAIN]})$|^$`);St("COMPARATOR",`^${$e[et.GTLT]}\\s*(${$e[et.FULLPLAIN]})$|^$`);St("COMPARATORTRIM",`(\\s*)${$e[et.GTLT]}\\s*(${$e[et.LOOSEPLAIN]}|${$e[et.XRANGEPLAIN]})`,!0);bA.comparatorTrimReplace="$1$2$3";St("HYPHENRANGE",`^\\s*(${$e[et.XRANGEPLAIN]})\\s+-\\s+(${$e[et.XRANGEPLAIN]})\\s*$`);St("HYPHENRANGELOOSE",`^\\s*(${$e[et.XRANGEPLAINLOOSE]})\\s+-\\s+(${$e[et.XRANGEPLAINLOOSE]})\\s*$`);St("STAR","(<|>)?=?\\s*\\*");St("GTE0","^\\s*>=\\s*0.0.0\\s*$");St("GTE0PRE","^\\s*>=\\s*0.0.0-0\\s*$")});var rd=w((eZe,mH)=>{var xde=["includePrerelease","loose","rtl"],Pde=r=>r?typeof r!="object"?{loose:!0}:xde.filter(e=>r[e]).reduce((e,t)=>(e[t]=!0,e),{}):{};mH.exports=Pde});var bI=w((tZe,yH)=>{var EH=/^[0-9]+$/,IH=(r,e)=>{let t=EH.test(r),i=EH.test(e);return t&&i&&(r=+r,e=+e),r===e?0:t&&!i?-1:i&&!t?1:rIH(e,r);yH.exports={compareIdentifiers:IH,rcompareIdentifiers:Dde}});var Li=w((rZe,bH)=>{var SI=td(),{MAX_LENGTH:wH,MAX_SAFE_INTEGER:vI}=ed(),{re:BH,t:QH}=Zl(),kde=rd(),{compareIdentifiers:id}=bI(),Un=class{constructor(e,t){if(t=kde(t),e instanceof Un){if(e.loose===!!t.loose&&e.includePrerelease===!!t.includePrerelease)return e;e=e.version}else if(typeof e!="string")throw new TypeError(`Invalid Version: ${e}`);if(e.length>wH)throw new TypeError(`version is longer than ${wH} characters`);SI("SemVer",e,t),this.options=t,this.loose=!!t.loose,this.includePrerelease=!!t.includePrerelease;let i=e.trim().match(t.loose?BH[QH.LOOSE]:BH[QH.FULL]);if(!i)throw new TypeError(`Invalid Version: ${e}`);if(this.raw=e,this.major=+i[1],this.minor=+i[2],this.patch=+i[3],this.major>vI||this.major<0)throw new TypeError("Invalid major version");if(this.minor>vI||this.minor<0)throw new TypeError("Invalid minor version");if(this.patch>vI||this.patch<0)throw new TypeError("Invalid patch version");i[4]?this.prerelease=i[4].split(".").map(n=>{if(/^[0-9]+$/.test(n)){let s=+n;if(s>=0&&s=0;)typeof this.prerelease[i]=="number"&&(this.prerelease[i]++,i=-2);i===-1&&this.prerelease.push(0)}t&&(this.prerelease[0]===t?isNaN(this.prerelease[1])&&(this.prerelease=[t,0]):this.prerelease=[t,0]);break;default:throw new Error(`invalid increment argument: ${e}`)}return this.format(),this.raw=this.version,this}};bH.exports=Un});var $l=w((iZe,PH)=>{var{MAX_LENGTH:Rde}=ed(),{re:SH,t:vH}=Zl(),xH=Li(),Fde=rd(),Nde=(r,e)=>{if(e=Fde(e),r instanceof xH)return r;if(typeof r!="string"||r.length>Rde||!(e.loose?SH[vH.LOOSE]:SH[vH.FULL]).test(r))return null;try{return new xH(r,e)}catch{return null}};PH.exports=Nde});var kH=w((nZe,DH)=>{var Lde=$l(),Tde=(r,e)=>{let t=Lde(r,e);return t?t.version:null};DH.exports=Tde});var FH=w((sZe,RH)=>{var Ode=$l(),Mde=(r,e)=>{let t=Ode(r.trim().replace(/^[=v]+/,""),e);return t?t.version:null};RH.exports=Mde});var LH=w((oZe,NH)=>{var Kde=Li(),Ude=(r,e,t,i)=>{typeof t=="string"&&(i=t,t=void 0);try{return new Kde(r,t).inc(e,i).version}catch{return null}};NH.exports=Ude});var ss=w((aZe,OH)=>{var TH=Li(),Hde=(r,e,t)=>new TH(r,t).compare(new TH(e,t));OH.exports=Hde});var xI=w((AZe,MH)=>{var Gde=ss(),Yde=(r,e,t)=>Gde(r,e,t)===0;MH.exports=Yde});var HH=w((lZe,UH)=>{var KH=$l(),jde=xI(),qde=(r,e)=>{if(jde(r,e))return null;{let t=KH(r),i=KH(e),n=t.prerelease.length||i.prerelease.length,s=n?"pre":"",o=n?"prerelease":"";for(let a in t)if((a==="major"||a==="minor"||a==="patch")&&t[a]!==i[a])return s+a;return o}};UH.exports=qde});var YH=w((cZe,GH)=>{var Jde=Li(),Wde=(r,e)=>new Jde(r,e).major;GH.exports=Wde});var qH=w((uZe,jH)=>{var zde=Li(),Vde=(r,e)=>new zde(r,e).minor;jH.exports=Vde});var WH=w((gZe,JH)=>{var Xde=Li(),_de=(r,e)=>new Xde(r,e).patch;JH.exports=_de});var VH=w((fZe,zH)=>{var Zde=$l(),$de=(r,e)=>{let t=Zde(r,e);return t&&t.prerelease.length?t.prerelease:null};zH.exports=$de});var _H=w((hZe,XH)=>{var eCe=ss(),tCe=(r,e,t)=>eCe(e,r,t);XH.exports=tCe});var $H=w((pZe,ZH)=>{var rCe=ss(),iCe=(r,e)=>rCe(r,e,!0);ZH.exports=iCe});var PI=w((dZe,tG)=>{var eG=Li(),nCe=(r,e,t)=>{let i=new eG(r,t),n=new eG(e,t);return i.compare(n)||i.compareBuild(n)};tG.exports=nCe});var iG=w((CZe,rG)=>{var sCe=PI(),oCe=(r,e)=>r.sort((t,i)=>sCe(t,i,e));rG.exports=oCe});var sG=w((mZe,nG)=>{var aCe=PI(),ACe=(r,e)=>r.sort((t,i)=>aCe(i,t,e));nG.exports=ACe});var nd=w((EZe,oG)=>{var lCe=ss(),cCe=(r,e,t)=>lCe(r,e,t)>0;oG.exports=cCe});var DI=w((IZe,aG)=>{var uCe=ss(),gCe=(r,e,t)=>uCe(r,e,t)<0;aG.exports=gCe});var rv=w((yZe,AG)=>{var fCe=ss(),hCe=(r,e,t)=>fCe(r,e,t)!==0;AG.exports=hCe});var kI=w((wZe,lG)=>{var pCe=ss(),dCe=(r,e,t)=>pCe(r,e,t)>=0;lG.exports=dCe});var RI=w((BZe,cG)=>{var CCe=ss(),mCe=(r,e,t)=>CCe(r,e,t)<=0;cG.exports=mCe});var iv=w((QZe,uG)=>{var ECe=xI(),ICe=rv(),yCe=nd(),wCe=kI(),BCe=DI(),QCe=RI(),bCe=(r,e,t,i)=>{switch(e){case"===":return typeof r=="object"&&(r=r.version),typeof t=="object"&&(t=t.version),r===t;case"!==":return typeof r=="object"&&(r=r.version),typeof t=="object"&&(t=t.version),r!==t;case"":case"=":case"==":return ECe(r,t,i);case"!=":return ICe(r,t,i);case">":return yCe(r,t,i);case">=":return wCe(r,t,i);case"<":return BCe(r,t,i);case"<=":return QCe(r,t,i);default:throw new TypeError(`Invalid operator: ${e}`)}};uG.exports=bCe});var fG=w((bZe,gG)=>{var SCe=Li(),vCe=$l(),{re:FI,t:NI}=Zl(),xCe=(r,e)=>{if(r instanceof SCe)return r;if(typeof r=="number"&&(r=String(r)),typeof r!="string")return null;e=e||{};let t=null;if(!e.rtl)t=r.match(FI[NI.COERCE]);else{let i;for(;(i=FI[NI.COERCERTL].exec(r))&&(!t||t.index+t[0].length!==r.length);)(!t||i.index+i[0].length!==t.index+t[0].length)&&(t=i),FI[NI.COERCERTL].lastIndex=i.index+i[1].length+i[2].length;FI[NI.COERCERTL].lastIndex=-1}return t===null?null:vCe(`${t[2]}.${t[3]||"0"}.${t[4]||"0"}`,e)};gG.exports=xCe});var pG=w((SZe,hG)=>{"use strict";hG.exports=function(r){r.prototype[Symbol.iterator]=function*(){for(let e=this.head;e;e=e.next)yield e.value}}});var sd=w((vZe,dG)=>{"use strict";dG.exports=Ht;Ht.Node=ec;Ht.create=Ht;function Ht(r){var e=this;if(e instanceof Ht||(e=new Ht),e.tail=null,e.head=null,e.length=0,r&&typeof r.forEach=="function")r.forEach(function(n){e.push(n)});else if(arguments.length>0)for(var t=0,i=arguments.length;t1)t=e;else if(this.head)i=this.head.next,t=this.head.value;else throw new TypeError("Reduce of empty list with no initial value");for(var n=0;i!==null;n++)t=r(t,i.value,n),i=i.next;return t};Ht.prototype.reduceReverse=function(r,e){var t,i=this.tail;if(arguments.length>1)t=e;else if(this.tail)i=this.tail.prev,t=this.tail.value;else throw new TypeError("Reduce of empty list with no initial value");for(var n=this.length-1;i!==null;n--)t=r(t,i.value,n),i=i.prev;return t};Ht.prototype.toArray=function(){for(var r=new Array(this.length),e=0,t=this.head;t!==null;e++)r[e]=t.value,t=t.next;return r};Ht.prototype.toArrayReverse=function(){for(var r=new Array(this.length),e=0,t=this.tail;t!==null;e++)r[e]=t.value,t=t.prev;return r};Ht.prototype.slice=function(r,e){e=e||this.length,e<0&&(e+=this.length),r=r||0,r<0&&(r+=this.length);var t=new Ht;if(ethis.length&&(e=this.length);for(var i=0,n=this.head;n!==null&&ithis.length&&(e=this.length);for(var i=this.length,n=this.tail;n!==null&&i>e;i--)n=n.prev;for(;n!==null&&i>r;i--,n=n.prev)t.push(n.value);return t};Ht.prototype.splice=function(r,e,...t){r>this.length&&(r=this.length-1),r<0&&(r=this.length+r);for(var i=0,n=this.head;n!==null&&i{"use strict";var RCe=sd(),tc=Symbol("max"),Ia=Symbol("length"),Og=Symbol("lengthCalculator"),ad=Symbol("allowStale"),rc=Symbol("maxAge"),Ea=Symbol("dispose"),CG=Symbol("noDisposeOnSet"),di=Symbol("lruList"),Ws=Symbol("cache"),EG=Symbol("updateAgeOnGet"),nv=()=>1,ov=class{constructor(e){if(typeof e=="number"&&(e={max:e}),e||(e={}),e.max&&(typeof e.max!="number"||e.max<0))throw new TypeError("max must be a non-negative number");let t=this[tc]=e.max||1/0,i=e.length||nv;if(this[Og]=typeof i!="function"?nv:i,this[ad]=e.stale||!1,e.maxAge&&typeof e.maxAge!="number")throw new TypeError("maxAge must be a number");this[rc]=e.maxAge||0,this[Ea]=e.dispose,this[CG]=e.noDisposeOnSet||!1,this[EG]=e.updateAgeOnGet||!1,this.reset()}set max(e){if(typeof e!="number"||e<0)throw new TypeError("max must be a non-negative number");this[tc]=e||1/0,od(this)}get max(){return this[tc]}set allowStale(e){this[ad]=!!e}get allowStale(){return this[ad]}set maxAge(e){if(typeof e!="number")throw new TypeError("maxAge must be a non-negative number");this[rc]=e,od(this)}get maxAge(){return this[rc]}set lengthCalculator(e){typeof e!="function"&&(e=nv),e!==this[Og]&&(this[Og]=e,this[Ia]=0,this[di].forEach(t=>{t.length=this[Og](t.value,t.key),this[Ia]+=t.length})),od(this)}get lengthCalculator(){return this[Og]}get length(){return this[Ia]}get itemCount(){return this[di].length}rforEach(e,t){t=t||this;for(let i=this[di].tail;i!==null;){let n=i.prev;mG(this,e,i,t),i=n}}forEach(e,t){t=t||this;for(let i=this[di].head;i!==null;){let n=i.next;mG(this,e,i,t),i=n}}keys(){return this[di].toArray().map(e=>e.key)}values(){return this[di].toArray().map(e=>e.value)}reset(){this[Ea]&&this[di]&&this[di].length&&this[di].forEach(e=>this[Ea](e.key,e.value)),this[Ws]=new Map,this[di]=new RCe,this[Ia]=0}dump(){return this[di].map(e=>LI(this,e)?!1:{k:e.key,v:e.value,e:e.now+(e.maxAge||0)}).toArray().filter(e=>e)}dumpLru(){return this[di]}set(e,t,i){if(i=i||this[rc],i&&typeof i!="number")throw new TypeError("maxAge must be a number");let n=i?Date.now():0,s=this[Og](t,e);if(this[Ws].has(e)){if(s>this[tc])return Mg(this,this[Ws].get(e)),!1;let l=this[Ws].get(e).value;return this[Ea]&&(this[CG]||this[Ea](e,l.value)),l.now=n,l.maxAge=i,l.value=t,this[Ia]+=s-l.length,l.length=s,this.get(e),od(this),!0}let o=new av(e,t,s,n,i);return o.length>this[tc]?(this[Ea]&&this[Ea](e,t),!1):(this[Ia]+=o.length,this[di].unshift(o),this[Ws].set(e,this[di].head),od(this),!0)}has(e){if(!this[Ws].has(e))return!1;let t=this[Ws].get(e).value;return!LI(this,t)}get(e){return sv(this,e,!0)}peek(e){return sv(this,e,!1)}pop(){let e=this[di].tail;return e?(Mg(this,e),e.value):null}del(e){Mg(this,this[Ws].get(e))}load(e){this.reset();let t=Date.now();for(let i=e.length-1;i>=0;i--){let n=e[i],s=n.e||0;if(s===0)this.set(n.k,n.v);else{let o=s-t;o>0&&this.set(n.k,n.v,o)}}}prune(){this[Ws].forEach((e,t)=>sv(this,t,!1))}},sv=(r,e,t)=>{let i=r[Ws].get(e);if(i){let n=i.value;if(LI(r,n)){if(Mg(r,i),!r[ad])return}else t&&(r[EG]&&(i.value.now=Date.now()),r[di].unshiftNode(i));return n.value}},LI=(r,e)=>{if(!e||!e.maxAge&&!r[rc])return!1;let t=Date.now()-e.now;return e.maxAge?t>e.maxAge:r[rc]&&t>r[rc]},od=r=>{if(r[Ia]>r[tc])for(let e=r[di].tail;r[Ia]>r[tc]&&e!==null;){let t=e.prev;Mg(r,e),e=t}},Mg=(r,e)=>{if(e){let t=e.value;r[Ea]&&r[Ea](t.key,t.value),r[Ia]-=t.length,r[Ws].delete(t.key),r[di].removeNode(e)}},av=class{constructor(e,t,i,n,s){this.key=e,this.value=t,this.length=i,this.now=n,this.maxAge=s||0}},mG=(r,e,t,i)=>{let n=t.value;LI(r,n)&&(Mg(r,t),r[ad]||(n=void 0)),n&&e.call(i,n.value,n.key,r)};IG.exports=ov});var os=w((PZe,bG)=>{var ic=class{constructor(e,t){if(t=NCe(t),e instanceof ic)return e.loose===!!t.loose&&e.includePrerelease===!!t.includePrerelease?e:new ic(e.raw,t);if(e instanceof Av)return this.raw=e.value,this.set=[[e]],this.format(),this;if(this.options=t,this.loose=!!t.loose,this.includePrerelease=!!t.includePrerelease,this.raw=e,this.set=e.split(/\s*\|\|\s*/).map(i=>this.parseRange(i.trim())).filter(i=>i.length),!this.set.length)throw new TypeError(`Invalid SemVer Range: ${e}`);if(this.set.length>1){let i=this.set[0];if(this.set=this.set.filter(n=>!BG(n[0])),this.set.length===0)this.set=[i];else if(this.set.length>1){for(let n of this.set)if(n.length===1&&KCe(n[0])){this.set=[n];break}}}this.format()}format(){return this.range=this.set.map(e=>e.join(" ").trim()).join("||").trim(),this.range}toString(){return this.range}parseRange(e){e=e.trim();let i=`parseRange:${Object.keys(this.options).join(",")}:${e}`,n=wG.get(i);if(n)return n;let s=this.options.loose,o=s?Ti[Bi.HYPHENRANGELOOSE]:Ti[Bi.HYPHENRANGE];e=e.replace(o,VCe(this.options.includePrerelease)),Gr("hyphen replace",e),e=e.replace(Ti[Bi.COMPARATORTRIM],TCe),Gr("comparator trim",e,Ti[Bi.COMPARATORTRIM]),e=e.replace(Ti[Bi.TILDETRIM],OCe),e=e.replace(Ti[Bi.CARETTRIM],MCe),e=e.split(/\s+/).join(" ");let a=s?Ti[Bi.COMPARATORLOOSE]:Ti[Bi.COMPARATOR],l=e.split(" ").map(f=>UCe(f,this.options)).join(" ").split(/\s+/).map(f=>zCe(f,this.options)).filter(this.options.loose?f=>!!f.match(a):()=>!0).map(f=>new Av(f,this.options)),c=l.length,u=new Map;for(let f of l){if(BG(f))return[f];u.set(f.value,f)}u.size>1&&u.has("")&&u.delete("");let g=[...u.values()];return wG.set(i,g),g}intersects(e,t){if(!(e instanceof ic))throw new TypeError("a Range is required");return this.set.some(i=>QG(i,t)&&e.set.some(n=>QG(n,t)&&i.every(s=>n.every(o=>s.intersects(o,t)))))}test(e){if(!e)return!1;if(typeof e=="string")try{e=new LCe(e,this.options)}catch{return!1}for(let t=0;tr.value==="<0.0.0-0",KCe=r=>r.value==="",QG=(r,e)=>{let t=!0,i=r.slice(),n=i.pop();for(;t&&i.length;)t=i.every(s=>n.intersects(s,e)),n=i.pop();return t},UCe=(r,e)=>(Gr("comp",r,e),r=YCe(r,e),Gr("caret",r),r=HCe(r,e),Gr("tildes",r),r=qCe(r,e),Gr("xrange",r),r=WCe(r,e),Gr("stars",r),r),Vi=r=>!r||r.toLowerCase()==="x"||r==="*",HCe=(r,e)=>r.trim().split(/\s+/).map(t=>GCe(t,e)).join(" "),GCe=(r,e)=>{let t=e.loose?Ti[Bi.TILDELOOSE]:Ti[Bi.TILDE];return r.replace(t,(i,n,s,o,a)=>{Gr("tilde",r,i,n,s,o,a);let l;return Vi(n)?l="":Vi(s)?l=`>=${n}.0.0 <${+n+1}.0.0-0`:Vi(o)?l=`>=${n}.${s}.0 <${n}.${+s+1}.0-0`:a?(Gr("replaceTilde pr",a),l=`>=${n}.${s}.${o}-${a} <${n}.${+s+1}.0-0`):l=`>=${n}.${s}.${o} <${n}.${+s+1}.0-0`,Gr("tilde return",l),l})},YCe=(r,e)=>r.trim().split(/\s+/).map(t=>jCe(t,e)).join(" "),jCe=(r,e)=>{Gr("caret",r,e);let t=e.loose?Ti[Bi.CARETLOOSE]:Ti[Bi.CARET],i=e.includePrerelease?"-0":"";return r.replace(t,(n,s,o,a,l)=>{Gr("caret",r,n,s,o,a,l);let c;return Vi(s)?c="":Vi(o)?c=`>=${s}.0.0${i} <${+s+1}.0.0-0`:Vi(a)?s==="0"?c=`>=${s}.${o}.0${i} <${s}.${+o+1}.0-0`:c=`>=${s}.${o}.0${i} <${+s+1}.0.0-0`:l?(Gr("replaceCaret pr",l),s==="0"?o==="0"?c=`>=${s}.${o}.${a}-${l} <${s}.${o}.${+a+1}-0`:c=`>=${s}.${o}.${a}-${l} <${s}.${+o+1}.0-0`:c=`>=${s}.${o}.${a}-${l} <${+s+1}.0.0-0`):(Gr("no pr"),s==="0"?o==="0"?c=`>=${s}.${o}.${a}${i} <${s}.${o}.${+a+1}-0`:c=`>=${s}.${o}.${a}${i} <${s}.${+o+1}.0-0`:c=`>=${s}.${o}.${a} <${+s+1}.0.0-0`),Gr("caret return",c),c})},qCe=(r,e)=>(Gr("replaceXRanges",r,e),r.split(/\s+/).map(t=>JCe(t,e)).join(" ")),JCe=(r,e)=>{r=r.trim();let t=e.loose?Ti[Bi.XRANGELOOSE]:Ti[Bi.XRANGE];return r.replace(t,(i,n,s,o,a,l)=>{Gr("xRange",r,i,n,s,o,a,l);let c=Vi(s),u=c||Vi(o),g=u||Vi(a),f=g;return n==="="&&f&&(n=""),l=e.includePrerelease?"-0":"",c?n===">"||n==="<"?i="<0.0.0-0":i="*":n&&f?(u&&(o=0),a=0,n===">"?(n=">=",u?(s=+s+1,o=0,a=0):(o=+o+1,a=0)):n==="<="&&(n="<",u?s=+s+1:o=+o+1),n==="<"&&(l="-0"),i=`${n+s}.${o}.${a}${l}`):u?i=`>=${s}.0.0${l} <${+s+1}.0.0-0`:g&&(i=`>=${s}.${o}.0${l} <${s}.${+o+1}.0-0`),Gr("xRange return",i),i})},WCe=(r,e)=>(Gr("replaceStars",r,e),r.trim().replace(Ti[Bi.STAR],"")),zCe=(r,e)=>(Gr("replaceGTE0",r,e),r.trim().replace(Ti[e.includePrerelease?Bi.GTE0PRE:Bi.GTE0],"")),VCe=r=>(e,t,i,n,s,o,a,l,c,u,g,f,h)=>(Vi(i)?t="":Vi(n)?t=`>=${i}.0.0${r?"-0":""}`:Vi(s)?t=`>=${i}.${n}.0${r?"-0":""}`:o?t=`>=${t}`:t=`>=${t}${r?"-0":""}`,Vi(c)?l="":Vi(u)?l=`<${+c+1}.0.0-0`:Vi(g)?l=`<${c}.${+u+1}.0-0`:f?l=`<=${c}.${u}.${g}-${f}`:r?l=`<${c}.${u}.${+g+1}-0`:l=`<=${l}`,`${t} ${l}`.trim()),XCe=(r,e,t)=>{for(let i=0;i0){let n=r[i].semver;if(n.major===e.major&&n.minor===e.minor&&n.patch===e.patch)return!0}return!1}return!0}});var Ad=w((DZe,DG)=>{var ld=Symbol("SemVer ANY"),Kg=class{static get ANY(){return ld}constructor(e,t){if(t=_Ce(t),e instanceof Kg){if(e.loose===!!t.loose)return e;e=e.value}cv("comparator",e,t),this.options=t,this.loose=!!t.loose,this.parse(e),this.semver===ld?this.value="":this.value=this.operator+this.semver.version,cv("comp",this)}parse(e){let t=this.options.loose?SG[vG.COMPARATORLOOSE]:SG[vG.COMPARATOR],i=e.match(t);if(!i)throw new TypeError(`Invalid comparator: ${e}`);this.operator=i[1]!==void 0?i[1]:"",this.operator==="="&&(this.operator=""),i[2]?this.semver=new xG(i[2],this.options.loose):this.semver=ld}toString(){return this.value}test(e){if(cv("Comparator.test",e,this.options.loose),this.semver===ld||e===ld)return!0;if(typeof e=="string")try{e=new xG(e,this.options)}catch{return!1}return lv(e,this.operator,this.semver,this.options)}intersects(e,t){if(!(e instanceof Kg))throw new TypeError("a Comparator is required");if((!t||typeof t!="object")&&(t={loose:!!t,includePrerelease:!1}),this.operator==="")return this.value===""?!0:new PG(e.value,t).test(this.value);if(e.operator==="")return e.value===""?!0:new PG(this.value,t).test(e.semver);let i=(this.operator===">="||this.operator===">")&&(e.operator===">="||e.operator===">"),n=(this.operator==="<="||this.operator==="<")&&(e.operator==="<="||e.operator==="<"),s=this.semver.version===e.semver.version,o=(this.operator===">="||this.operator==="<=")&&(e.operator===">="||e.operator==="<="),a=lv(this.semver,"<",e.semver,t)&&(this.operator===">="||this.operator===">")&&(e.operator==="<="||e.operator==="<"),l=lv(this.semver,">",e.semver,t)&&(this.operator==="<="||this.operator==="<")&&(e.operator===">="||e.operator===">");return i||n||s&&o||a||l}};DG.exports=Kg;var _Ce=rd(),{re:SG,t:vG}=Zl(),lv=iv(),cv=td(),xG=Li(),PG=os()});var cd=w((kZe,kG)=>{var ZCe=os(),$Ce=(r,e,t)=>{try{e=new ZCe(e,t)}catch{return!1}return e.test(r)};kG.exports=$Ce});var FG=w((RZe,RG)=>{var eme=os(),tme=(r,e)=>new eme(r,e).set.map(t=>t.map(i=>i.value).join(" ").trim().split(" "));RG.exports=tme});var LG=w((FZe,NG)=>{var rme=Li(),ime=os(),nme=(r,e,t)=>{let i=null,n=null,s=null;try{s=new ime(e,t)}catch{return null}return r.forEach(o=>{s.test(o)&&(!i||n.compare(o)===-1)&&(i=o,n=new rme(i,t))}),i};NG.exports=nme});var OG=w((NZe,TG)=>{var sme=Li(),ome=os(),ame=(r,e,t)=>{let i=null,n=null,s=null;try{s=new ome(e,t)}catch{return null}return r.forEach(o=>{s.test(o)&&(!i||n.compare(o)===1)&&(i=o,n=new sme(i,t))}),i};TG.exports=ame});var UG=w((LZe,KG)=>{var uv=Li(),Ame=os(),MG=nd(),lme=(r,e)=>{r=new Ame(r,e);let t=new uv("0.0.0");if(r.test(t)||(t=new uv("0.0.0-0"),r.test(t)))return t;t=null;for(let i=0;i{let a=new uv(o.semver.version);switch(o.operator){case">":a.prerelease.length===0?a.patch++:a.prerelease.push(0),a.raw=a.format();case"":case">=":(!s||MG(a,s))&&(s=a);break;case"<":case"<=":break;default:throw new Error(`Unexpected operation: ${o.operator}`)}}),s&&(!t||MG(t,s))&&(t=s)}return t&&r.test(t)?t:null};KG.exports=lme});var GG=w((TZe,HG)=>{var cme=os(),ume=(r,e)=>{try{return new cme(r,e).range||"*"}catch{return null}};HG.exports=ume});var TI=w((OZe,JG)=>{var gme=Li(),qG=Ad(),{ANY:fme}=qG,hme=os(),pme=cd(),YG=nd(),jG=DI(),dme=RI(),Cme=kI(),mme=(r,e,t,i)=>{r=new gme(r,i),e=new hme(e,i);let n,s,o,a,l;switch(t){case">":n=YG,s=dme,o=jG,a=">",l=">=";break;case"<":n=jG,s=Cme,o=YG,a="<",l="<=";break;default:throw new TypeError('Must provide a hilo val of "<" or ">"')}if(pme(r,e,i))return!1;for(let c=0;c{h.semver===fme&&(h=new qG(">=0.0.0")),g=g||h,f=f||h,n(h.semver,g.semver,i)?g=h:o(h.semver,f.semver,i)&&(f=h)}),g.operator===a||g.operator===l||(!f.operator||f.operator===a)&&s(r,f.semver))return!1;if(f.operator===l&&o(r,f.semver))return!1}return!0};JG.exports=mme});var zG=w((MZe,WG)=>{var Eme=TI(),Ime=(r,e,t)=>Eme(r,e,">",t);WG.exports=Ime});var XG=w((KZe,VG)=>{var yme=TI(),wme=(r,e,t)=>yme(r,e,"<",t);VG.exports=wme});var $G=w((UZe,ZG)=>{var _G=os(),Bme=(r,e,t)=>(r=new _G(r,t),e=new _G(e,t),r.intersects(e));ZG.exports=Bme});var tY=w((HZe,eY)=>{var Qme=cd(),bme=ss();eY.exports=(r,e,t)=>{let i=[],n=null,s=null,o=r.sort((u,g)=>bme(u,g,t));for(let u of o)Qme(u,e,t)?(s=u,n||(n=u)):(s&&i.push([n,s]),s=null,n=null);n&&i.push([n,null]);let a=[];for(let[u,g]of i)u===g?a.push(u):!g&&u===o[0]?a.push("*"):g?u===o[0]?a.push(`<=${g}`):a.push(`${u} - ${g}`):a.push(`>=${u}`);let l=a.join(" || "),c=typeof e.raw=="string"?e.raw:String(e);return l.length{var rY=os(),OI=Ad(),{ANY:gv}=OI,ud=cd(),fv=ss(),Sme=(r,e,t={})=>{if(r===e)return!0;r=new rY(r,t),e=new rY(e,t);let i=!1;e:for(let n of r.set){for(let s of e.set){let o=vme(n,s,t);if(i=i||o!==null,o)continue e}if(i)return!1}return!0},vme=(r,e,t)=>{if(r===e)return!0;if(r.length===1&&r[0].semver===gv){if(e.length===1&&e[0].semver===gv)return!0;t.includePrerelease?r=[new OI(">=0.0.0-0")]:r=[new OI(">=0.0.0")]}if(e.length===1&&e[0].semver===gv){if(t.includePrerelease)return!0;e=[new OI(">=0.0.0")]}let i=new Set,n,s;for(let h of r)h.operator===">"||h.operator===">="?n=iY(n,h,t):h.operator==="<"||h.operator==="<="?s=nY(s,h,t):i.add(h.semver);if(i.size>1)return null;let o;if(n&&s){if(o=fv(n.semver,s.semver,t),o>0)return null;if(o===0&&(n.operator!==">="||s.operator!=="<="))return null}for(let h of i){if(n&&!ud(h,String(n),t)||s&&!ud(h,String(s),t))return null;for(let p of e)if(!ud(h,String(p),t))return!1;return!0}let a,l,c,u,g=s&&!t.includePrerelease&&s.semver.prerelease.length?s.semver:!1,f=n&&!t.includePrerelease&&n.semver.prerelease.length?n.semver:!1;g&&g.prerelease.length===1&&s.operator==="<"&&g.prerelease[0]===0&&(g=!1);for(let h of e){if(u=u||h.operator===">"||h.operator===">=",c=c||h.operator==="<"||h.operator==="<=",n){if(f&&h.semver.prerelease&&h.semver.prerelease.length&&h.semver.major===f.major&&h.semver.minor===f.minor&&h.semver.patch===f.patch&&(f=!1),h.operator===">"||h.operator===">="){if(a=iY(n,h,t),a===h&&a!==n)return!1}else if(n.operator===">="&&!ud(n.semver,String(h),t))return!1}if(s){if(g&&h.semver.prerelease&&h.semver.prerelease.length&&h.semver.major===g.major&&h.semver.minor===g.minor&&h.semver.patch===g.patch&&(g=!1),h.operator==="<"||h.operator==="<="){if(l=nY(s,h,t),l===h&&l!==s)return!1}else if(s.operator==="<="&&!ud(s.semver,String(h),t))return!1}if(!h.operator&&(s||n)&&o!==0)return!1}return!(n&&c&&!s&&o!==0||s&&u&&!n&&o!==0||f||g)},iY=(r,e,t)=>{if(!r)return e;let i=fv(r.semver,e.semver,t);return i>0?r:i<0||e.operator===">"&&r.operator===">="?e:r},nY=(r,e,t)=>{if(!r)return e;let i=fv(r.semver,e.semver,t);return i<0?r:i>0||e.operator==="<"&&r.operator==="<="?e:r};sY.exports=Sme});var Xr=w((YZe,aY)=>{var hv=Zl();aY.exports={re:hv.re,src:hv.src,tokens:hv.t,SEMVER_SPEC_VERSION:ed().SEMVER_SPEC_VERSION,SemVer:Li(),compareIdentifiers:bI().compareIdentifiers,rcompareIdentifiers:bI().rcompareIdentifiers,parse:$l(),valid:kH(),clean:FH(),inc:LH(),diff:HH(),major:YH(),minor:qH(),patch:WH(),prerelease:VH(),compare:ss(),rcompare:_H(),compareLoose:$H(),compareBuild:PI(),sort:iG(),rsort:sG(),gt:nd(),lt:DI(),eq:xI(),neq:rv(),gte:kI(),lte:RI(),cmp:iv(),coerce:fG(),Comparator:Ad(),Range:os(),satisfies:cd(),toComparators:FG(),maxSatisfying:LG(),minSatisfying:OG(),minVersion:UG(),validRange:GG(),outside:TI(),gtr:zG(),ltr:XG(),intersects:$G(),simplifyRange:tY(),subset:oY()}});var pv=w(MI=>{"use strict";Object.defineProperty(MI,"__esModule",{value:!0});MI.VERSION=void 0;MI.VERSION="9.1.0"});var Gt=w((exports,module)=>{"use strict";var __spreadArray=exports&&exports.__spreadArray||function(r,e,t){if(t||arguments.length===2)for(var i=0,n=e.length,s;i{(function(r,e){typeof define=="function"&&define.amd?define([],e):typeof KI=="object"&&KI.exports?KI.exports=e():r.regexpToAst=e()})(typeof self<"u"?self:AY,function(){function r(){}r.prototype.saveState=function(){return{idx:this.idx,input:this.input,groupIdx:this.groupIdx}},r.prototype.restoreState=function(p){this.idx=p.idx,this.input=p.input,this.groupIdx=p.groupIdx},r.prototype.pattern=function(p){this.idx=0,this.input=p,this.groupIdx=0,this.consumeChar("/");var C=this.disjunction();this.consumeChar("/");for(var y={type:"Flags",loc:{begin:this.idx,end:p.length},global:!1,ignoreCase:!1,multiLine:!1,unicode:!1,sticky:!1};this.isRegExpFlag();)switch(this.popChar()){case"g":o(y,"global");break;case"i":o(y,"ignoreCase");break;case"m":o(y,"multiLine");break;case"u":o(y,"unicode");break;case"y":o(y,"sticky");break}if(this.idx!==this.input.length)throw Error("Redundant input: "+this.input.substring(this.idx));return{type:"Pattern",flags:y,value:C,loc:this.loc(0)}},r.prototype.disjunction=function(){var p=[],C=this.idx;for(p.push(this.alternative());this.peekChar()==="|";)this.consumeChar("|"),p.push(this.alternative());return{type:"Disjunction",value:p,loc:this.loc(C)}},r.prototype.alternative=function(){for(var p=[],C=this.idx;this.isTerm();)p.push(this.term());return{type:"Alternative",value:p,loc:this.loc(C)}},r.prototype.term=function(){return this.isAssertion()?this.assertion():this.atom()},r.prototype.assertion=function(){var p=this.idx;switch(this.popChar()){case"^":return{type:"StartAnchor",loc:this.loc(p)};case"$":return{type:"EndAnchor",loc:this.loc(p)};case"\\":switch(this.popChar()){case"b":return{type:"WordBoundary",loc:this.loc(p)};case"B":return{type:"NonWordBoundary",loc:this.loc(p)}}throw Error("Invalid Assertion Escape");case"(":this.consumeChar("?");var C;switch(this.popChar()){case"=":C="Lookahead";break;case"!":C="NegativeLookahead";break}a(C);var y=this.disjunction();return this.consumeChar(")"),{type:C,value:y,loc:this.loc(p)}}l()},r.prototype.quantifier=function(p){var C,y=this.idx;switch(this.popChar()){case"*":C={atLeast:0,atMost:1/0};break;case"+":C={atLeast:1,atMost:1/0};break;case"?":C={atLeast:0,atMost:1};break;case"{":var B=this.integerIncludingZero();switch(this.popChar()){case"}":C={atLeast:B,atMost:B};break;case",":var v;this.isDigit()?(v=this.integerIncludingZero(),C={atLeast:B,atMost:v}):C={atLeast:B,atMost:1/0},this.consumeChar("}");break}if(p===!0&&C===void 0)return;a(C);break}if(!(p===!0&&C===void 0))return a(C),this.peekChar(0)==="?"?(this.consumeChar("?"),C.greedy=!1):C.greedy=!0,C.type="Quantifier",C.loc=this.loc(y),C},r.prototype.atom=function(){var p,C=this.idx;switch(this.peekChar()){case".":p=this.dotAll();break;case"\\":p=this.atomEscape();break;case"[":p=this.characterClass();break;case"(":p=this.group();break}return p===void 0&&this.isPatternCharacter()&&(p=this.patternCharacter()),a(p),p.loc=this.loc(C),this.isQuantifier()&&(p.quantifier=this.quantifier()),p},r.prototype.dotAll=function(){return this.consumeChar("."),{type:"Set",complement:!0,value:[n(` -`),n("\r"),n("\u2028"),n("\u2029")]}},r.prototype.atomEscape=function(){switch(this.consumeChar("\\"),this.peekChar()){case"1":case"2":case"3":case"4":case"5":case"6":case"7":case"8":case"9":return this.decimalEscapeAtom();case"d":case"D":case"s":case"S":case"w":case"W":return this.characterClassEscape();case"f":case"n":case"r":case"t":case"v":return this.controlEscapeAtom();case"c":return this.controlLetterEscapeAtom();case"0":return this.nulCharacterAtom();case"x":return this.hexEscapeSequenceAtom();case"u":return this.regExpUnicodeEscapeSequenceAtom();default:return this.identityEscapeAtom()}},r.prototype.decimalEscapeAtom=function(){var p=this.positiveInteger();return{type:"GroupBackReference",value:p}},r.prototype.characterClassEscape=function(){var p,C=!1;switch(this.popChar()){case"d":p=u;break;case"D":p=u,C=!0;break;case"s":p=f;break;case"S":p=f,C=!0;break;case"w":p=g;break;case"W":p=g,C=!0;break}return a(p),{type:"Set",value:p,complement:C}},r.prototype.controlEscapeAtom=function(){var p;switch(this.popChar()){case"f":p=n("\f");break;case"n":p=n(` -`);break;case"r":p=n("\r");break;case"t":p=n(" ");break;case"v":p=n("\v");break}return a(p),{type:"Character",value:p}},r.prototype.controlLetterEscapeAtom=function(){this.consumeChar("c");var p=this.popChar();if(/[a-zA-Z]/.test(p)===!1)throw Error("Invalid ");var C=p.toUpperCase().charCodeAt(0)-64;return{type:"Character",value:C}},r.prototype.nulCharacterAtom=function(){return this.consumeChar("0"),{type:"Character",value:n("\0")}},r.prototype.hexEscapeSequenceAtom=function(){return this.consumeChar("x"),this.parseHexDigits(2)},r.prototype.regExpUnicodeEscapeSequenceAtom=function(){return this.consumeChar("u"),this.parseHexDigits(4)},r.prototype.identityEscapeAtom=function(){var p=this.popChar();return{type:"Character",value:n(p)}},r.prototype.classPatternCharacterAtom=function(){switch(this.peekChar()){case` -`:case"\r":case"\u2028":case"\u2029":case"\\":case"]":throw Error("TBD");default:var p=this.popChar();return{type:"Character",value:n(p)}}},r.prototype.characterClass=function(){var p=[],C=!1;for(this.consumeChar("["),this.peekChar(0)==="^"&&(this.consumeChar("^"),C=!0);this.isClassAtom();){var y=this.classAtom(),B=y.type==="Character";if(B&&this.isRangeDash()){this.consumeChar("-");var v=this.classAtom(),D=v.type==="Character";if(D){if(v.value=this.input.length)throw Error("Unexpected end of input");this.idx++},r.prototype.loc=function(p){return{begin:p,end:this.idx}};var e=/[0-9a-fA-F]/,t=/[0-9]/,i=/[1-9]/;function n(p){return p.charCodeAt(0)}function s(p,C){p.length!==void 0?p.forEach(function(y){C.push(y)}):C.push(p)}function o(p,C){if(p[C]===!0)throw"duplicate flag "+C;p[C]=!0}function a(p){if(p===void 0)throw Error("Internal Error - Should never get here!")}function l(){throw Error("Internal Error - Should never get here!")}var c,u=[];for(c=n("0");c<=n("9");c++)u.push(c);var g=[n("_")].concat(u);for(c=n("a");c<=n("z");c++)g.push(c);for(c=n("A");c<=n("Z");c++)g.push(c);var f=[n(" "),n("\f"),n(` -`),n("\r"),n(" "),n("\v"),n(" "),n("\xA0"),n("\u1680"),n("\u2000"),n("\u2001"),n("\u2002"),n("\u2003"),n("\u2004"),n("\u2005"),n("\u2006"),n("\u2007"),n("\u2008"),n("\u2009"),n("\u200A"),n("\u2028"),n("\u2029"),n("\u202F"),n("\u205F"),n("\u3000"),n("\uFEFF")];function h(){}return h.prototype.visitChildren=function(p){for(var C in p){var y=p[C];p.hasOwnProperty(C)&&(y.type!==void 0?this.visit(y):Array.isArray(y)&&y.forEach(function(B){this.visit(B)},this))}},h.prototype.visit=function(p){switch(p.type){case"Pattern":this.visitPattern(p);break;case"Flags":this.visitFlags(p);break;case"Disjunction":this.visitDisjunction(p);break;case"Alternative":this.visitAlternative(p);break;case"StartAnchor":this.visitStartAnchor(p);break;case"EndAnchor":this.visitEndAnchor(p);break;case"WordBoundary":this.visitWordBoundary(p);break;case"NonWordBoundary":this.visitNonWordBoundary(p);break;case"Lookahead":this.visitLookahead(p);break;case"NegativeLookahead":this.visitNegativeLookahead(p);break;case"Character":this.visitCharacter(p);break;case"Set":this.visitSet(p);break;case"Group":this.visitGroup(p);break;case"GroupBackReference":this.visitGroupBackReference(p);break;case"Quantifier":this.visitQuantifier(p);break}this.visitChildren(p)},h.prototype.visitPattern=function(p){},h.prototype.visitFlags=function(p){},h.prototype.visitDisjunction=function(p){},h.prototype.visitAlternative=function(p){},h.prototype.visitStartAnchor=function(p){},h.prototype.visitEndAnchor=function(p){},h.prototype.visitWordBoundary=function(p){},h.prototype.visitNonWordBoundary=function(p){},h.prototype.visitLookahead=function(p){},h.prototype.visitNegativeLookahead=function(p){},h.prototype.visitCharacter=function(p){},h.prototype.visitSet=function(p){},h.prototype.visitGroup=function(p){},h.prototype.visitGroupBackReference=function(p){},h.prototype.visitQuantifier=function(p){},{RegExpParser:r,BaseRegExpVisitor:h,VERSION:"0.5.0"}})});var GI=w(Ug=>{"use strict";Object.defineProperty(Ug,"__esModule",{value:!0});Ug.clearRegExpParserCache=Ug.getRegExpAst=void 0;var xme=UI(),HI={},Pme=new xme.RegExpParser;function Dme(r){var e=r.toString();if(HI.hasOwnProperty(e))return HI[e];var t=Pme.pattern(e);return HI[e]=t,t}Ug.getRegExpAst=Dme;function kme(){HI={}}Ug.clearRegExpParserCache=kme});var fY=w(pn=>{"use strict";var Rme=pn&&pn.__extends||function(){var r=function(e,t){return r=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(i,n){i.__proto__=n}||function(i,n){for(var s in n)Object.prototype.hasOwnProperty.call(n,s)&&(i[s]=n[s])},r(e,t)};return function(e,t){if(typeof t!="function"&&t!==null)throw new TypeError("Class extends value "+String(t)+" is not a constructor or null");r(e,t);function i(){this.constructor=e}e.prototype=t===null?Object.create(t):(i.prototype=t.prototype,new i)}}();Object.defineProperty(pn,"__esModule",{value:!0});pn.canMatchCharCode=pn.firstCharOptimizedIndices=pn.getOptimizedStartCodesIndices=pn.failedOptimizationPrefixMsg=void 0;var cY=UI(),as=Gt(),uY=GI(),ya=Cv(),gY="Complement Sets are not supported for first char optimization";pn.failedOptimizationPrefixMsg=`Unable to use "first char" lexer optimizations: -`;function Fme(r,e){e===void 0&&(e=!1);try{var t=(0,uY.getRegExpAst)(r),i=jI(t.value,{},t.flags.ignoreCase);return i}catch(s){if(s.message===gY)e&&(0,as.PRINT_WARNING)(""+pn.failedOptimizationPrefixMsg+(" Unable to optimize: < "+r.toString()+` > -`)+` Complement Sets cannot be automatically optimized. - This will disable the lexer's first char optimizations. - See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#COMPLEMENT for details.`);else{var n="";e&&(n=` - This will disable the lexer's first char optimizations. - See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#REGEXP_PARSING for details.`),(0,as.PRINT_ERROR)(pn.failedOptimizationPrefixMsg+` -`+(" Failed parsing: < "+r.toString()+` > -`)+(" Using the regexp-to-ast library version: "+cY.VERSION+` -`)+" Please open an issue at: https://github.com/bd82/regexp-to-ast/issues"+n)}}return[]}pn.getOptimizedStartCodesIndices=Fme;function jI(r,e,t){switch(r.type){case"Disjunction":for(var i=0;i=ya.minOptimizationVal)for(var f=u.from>=ya.minOptimizationVal?u.from:ya.minOptimizationVal,h=u.to,p=(0,ya.charCodeToOptimizedIndex)(f),C=(0,ya.charCodeToOptimizedIndex)(h),y=p;y<=C;y++)e[y]=y}}});break;case"Group":jI(o.value,e,t);break;default:throw Error("Non Exhaustive Match")}var a=o.quantifier!==void 0&&o.quantifier.atLeast===0;if(o.type==="Group"&&dv(o)===!1||o.type!=="Group"&&a===!1)break}break;default:throw Error("non exhaustive match!")}return(0,as.values)(e)}pn.firstCharOptimizedIndices=jI;function YI(r,e,t){var i=(0,ya.charCodeToOptimizedIndex)(r);e[i]=i,t===!0&&Nme(r,e)}function Nme(r,e){var t=String.fromCharCode(r),i=t.toUpperCase();if(i!==t){var n=(0,ya.charCodeToOptimizedIndex)(i.charCodeAt(0));e[n]=n}else{var s=t.toLowerCase();if(s!==t){var n=(0,ya.charCodeToOptimizedIndex)(s.charCodeAt(0));e[n]=n}}}function lY(r,e){return(0,as.find)(r.value,function(t){if(typeof t=="number")return(0,as.contains)(e,t);var i=t;return(0,as.find)(e,function(n){return i.from<=n&&n<=i.to})!==void 0})}function dv(r){return r.quantifier&&r.quantifier.atLeast===0?!0:r.value?(0,as.isArray)(r.value)?(0,as.every)(r.value,dv):dv(r.value):!1}var Lme=function(r){Rme(e,r);function e(t){var i=r.call(this)||this;return i.targetCharCodes=t,i.found=!1,i}return e.prototype.visitChildren=function(t){if(this.found!==!0){switch(t.type){case"Lookahead":this.visitLookahead(t);return;case"NegativeLookahead":this.visitNegativeLookahead(t);return}r.prototype.visitChildren.call(this,t)}},e.prototype.visitCharacter=function(t){(0,as.contains)(this.targetCharCodes,t.value)&&(this.found=!0)},e.prototype.visitSet=function(t){t.complement?lY(t,this.targetCharCodes)===void 0&&(this.found=!0):lY(t,this.targetCharCodes)!==void 0&&(this.found=!0)},e}(cY.BaseRegExpVisitor);function Tme(r,e){if(e instanceof RegExp){var t=(0,uY.getRegExpAst)(e),i=new Lme(r);return i.visit(t),i.found}else return(0,as.find)(e,function(n){return(0,as.contains)(r,n.charCodeAt(0))})!==void 0}pn.canMatchCharCode=Tme});var Cv=w(Ve=>{"use strict";var hY=Ve&&Ve.__extends||function(){var r=function(e,t){return r=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(i,n){i.__proto__=n}||function(i,n){for(var s in n)Object.prototype.hasOwnProperty.call(n,s)&&(i[s]=n[s])},r(e,t)};return function(e,t){if(typeof t!="function"&&t!==null)throw new TypeError("Class extends value "+String(t)+" is not a constructor or null");r(e,t);function i(){this.constructor=e}e.prototype=t===null?Object.create(t):(i.prototype=t.prototype,new i)}}();Object.defineProperty(Ve,"__esModule",{value:!0});Ve.charCodeToOptimizedIndex=Ve.minOptimizationVal=Ve.buildLineBreakIssueMessage=Ve.LineTerminatorOptimizedTester=Ve.isShortPattern=Ve.isCustomPattern=Ve.cloneEmptyGroups=Ve.performWarningRuntimeChecks=Ve.performRuntimeChecks=Ve.addStickyFlag=Ve.addStartOfInput=Ve.findUnreachablePatterns=Ve.findModesThatDoNotExist=Ve.findInvalidGroupType=Ve.findDuplicatePatterns=Ve.findUnsupportedFlags=Ve.findStartOfInputAnchor=Ve.findEmptyMatchRegExps=Ve.findEndOfInputAnchor=Ve.findInvalidPatterns=Ve.findMissingPatterns=Ve.validatePatterns=Ve.analyzeTokenTypes=Ve.enableSticky=Ve.disableSticky=Ve.SUPPORT_STICKY=Ve.MODES=Ve.DEFAULT_MODE=void 0;var pY=UI(),ir=gd(),xe=Gt(),Hg=fY(),dY=GI(),So="PATTERN";Ve.DEFAULT_MODE="defaultMode";Ve.MODES="modes";Ve.SUPPORT_STICKY=typeof new RegExp("(?:)").sticky=="boolean";function Ome(){Ve.SUPPORT_STICKY=!1}Ve.disableSticky=Ome;function Mme(){Ve.SUPPORT_STICKY=!0}Ve.enableSticky=Mme;function Kme(r,e){e=(0,xe.defaults)(e,{useSticky:Ve.SUPPORT_STICKY,debug:!1,safeMode:!1,positionTracking:"full",lineTerminatorCharacters:["\r",` -`],tracer:function(v,D){return D()}});var t=e.tracer;t("initCharCodeToOptimizedIndexMap",function(){Vme()});var i;t("Reject Lexer.NA",function(){i=(0,xe.reject)(r,function(v){return v[So]===ir.Lexer.NA})});var n=!1,s;t("Transform Patterns",function(){n=!1,s=(0,xe.map)(i,function(v){var D=v[So];if((0,xe.isRegExp)(D)){var L=D.source;return L.length===1&&L!=="^"&&L!=="$"&&L!=="."&&!D.ignoreCase?L:L.length===2&&L[0]==="\\"&&!(0,xe.contains)(["d","D","s","S","t","r","n","t","0","c","b","B","f","v","w","W"],L[1])?L[1]:e.useSticky?Iv(D):Ev(D)}else{if((0,xe.isFunction)(D))return n=!0,{exec:D};if((0,xe.has)(D,"exec"))return n=!0,D;if(typeof D=="string"){if(D.length===1)return D;var H=D.replace(/[\\^$.*+?()[\]{}|]/g,"\\$&"),j=new RegExp(H);return e.useSticky?Iv(j):Ev(j)}else throw Error("non exhaustive match")}})});var o,a,l,c,u;t("misc mapping",function(){o=(0,xe.map)(i,function(v){return v.tokenTypeIdx}),a=(0,xe.map)(i,function(v){var D=v.GROUP;if(D!==ir.Lexer.SKIPPED){if((0,xe.isString)(D))return D;if((0,xe.isUndefined)(D))return!1;throw Error("non exhaustive match")}}),l=(0,xe.map)(i,function(v){var D=v.LONGER_ALT;if(D){var L=(0,xe.isArray)(D)?(0,xe.map)(D,function(H){return(0,xe.indexOf)(i,H)}):[(0,xe.indexOf)(i,D)];return L}}),c=(0,xe.map)(i,function(v){return v.PUSH_MODE}),u=(0,xe.map)(i,function(v){return(0,xe.has)(v,"POP_MODE")})});var g;t("Line Terminator Handling",function(){var v=DY(e.lineTerminatorCharacters);g=(0,xe.map)(i,function(D){return!1}),e.positionTracking!=="onlyOffset"&&(g=(0,xe.map)(i,function(D){if((0,xe.has)(D,"LINE_BREAKS"))return D.LINE_BREAKS;if(xY(D,v)===!1)return(0,Hg.canMatchCharCode)(v,D.PATTERN)}))});var f,h,p,C;t("Misc Mapping #2",function(){f=(0,xe.map)(i,wv),h=(0,xe.map)(s,vY),p=(0,xe.reduce)(i,function(v,D){var L=D.GROUP;return(0,xe.isString)(L)&&L!==ir.Lexer.SKIPPED&&(v[L]=[]),v},{}),C=(0,xe.map)(s,function(v,D){return{pattern:s[D],longerAlt:l[D],canLineTerminator:g[D],isCustom:f[D],short:h[D],group:a[D],push:c[D],pop:u[D],tokenTypeIdx:o[D],tokenType:i[D]}})});var y=!0,B=[];return e.safeMode||t("First Char Optimization",function(){B=(0,xe.reduce)(i,function(v,D,L){if(typeof D.PATTERN=="string"){var H=D.PATTERN.charCodeAt(0),j=yv(H);mv(v,j,C[L])}else if((0,xe.isArray)(D.START_CHARS_HINT)){var $;(0,xe.forEach)(D.START_CHARS_HINT,function(W){var Z=typeof W=="string"?W.charCodeAt(0):W,A=yv(Z);$!==A&&($=A,mv(v,A,C[L]))})}else if((0,xe.isRegExp)(D.PATTERN))if(D.PATTERN.unicode)y=!1,e.ensureOptimizations&&(0,xe.PRINT_ERROR)(""+Hg.failedOptimizationPrefixMsg+(" Unable to analyze < "+D.PATTERN.toString()+` > pattern. -`)+` The regexp unicode flag is not currently supported by the regexp-to-ast library. - This will disable the lexer's first char optimizations. - For details See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#UNICODE_OPTIMIZE`);else{var V=(0,Hg.getOptimizedStartCodesIndices)(D.PATTERN,e.ensureOptimizations);(0,xe.isEmpty)(V)&&(y=!1),(0,xe.forEach)(V,function(W){mv(v,W,C[L])})}else e.ensureOptimizations&&(0,xe.PRINT_ERROR)(""+Hg.failedOptimizationPrefixMsg+(" TokenType: <"+D.name+`> is using a custom token pattern without providing parameter. -`)+` This will disable the lexer's first char optimizations. - For details See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#CUSTOM_OPTIMIZE`),y=!1;return v},[])}),t("ArrayPacking",function(){B=(0,xe.packArray)(B)}),{emptyGroups:p,patternIdxToConfig:C,charCodeToPatternIdxToConfig:B,hasCustom:n,canBeOptimized:y}}Ve.analyzeTokenTypes=Kme;function Ume(r,e){var t=[],i=CY(r);t=t.concat(i.errors);var n=mY(i.valid),s=n.valid;return t=t.concat(n.errors),t=t.concat(Hme(s)),t=t.concat(QY(s)),t=t.concat(bY(s,e)),t=t.concat(SY(s)),t}Ve.validatePatterns=Ume;function Hme(r){var e=[],t=(0,xe.filter)(r,function(i){return(0,xe.isRegExp)(i[So])});return e=e.concat(EY(t)),e=e.concat(yY(t)),e=e.concat(wY(t)),e=e.concat(BY(t)),e=e.concat(IY(t)),e}function CY(r){var e=(0,xe.filter)(r,function(n){return!(0,xe.has)(n,So)}),t=(0,xe.map)(e,function(n){return{message:"Token Type: ->"+n.name+"<- missing static 'PATTERN' property",type:ir.LexerDefinitionErrorType.MISSING_PATTERN,tokenTypes:[n]}}),i=(0,xe.difference)(r,e);return{errors:t,valid:i}}Ve.findMissingPatterns=CY;function mY(r){var e=(0,xe.filter)(r,function(n){var s=n[So];return!(0,xe.isRegExp)(s)&&!(0,xe.isFunction)(s)&&!(0,xe.has)(s,"exec")&&!(0,xe.isString)(s)}),t=(0,xe.map)(e,function(n){return{message:"Token Type: ->"+n.name+"<- static 'PATTERN' can only be a RegExp, a Function matching the {CustomPatternMatcherFunc} type or an Object matching the {ICustomPattern} interface.",type:ir.LexerDefinitionErrorType.INVALID_PATTERN,tokenTypes:[n]}}),i=(0,xe.difference)(r,e);return{errors:t,valid:i}}Ve.findInvalidPatterns=mY;var Gme=/[^\\][\$]/;function EY(r){var e=function(n){hY(s,n);function s(){var o=n!==null&&n.apply(this,arguments)||this;return o.found=!1,o}return s.prototype.visitEndAnchor=function(o){this.found=!0},s}(pY.BaseRegExpVisitor),t=(0,xe.filter)(r,function(n){var s=n[So];try{var o=(0,dY.getRegExpAst)(s),a=new e;return a.visit(o),a.found}catch{return Gme.test(s.source)}}),i=(0,xe.map)(t,function(n){return{message:`Unexpected RegExp Anchor Error: - Token Type: ->`+n.name+`<- static 'PATTERN' cannot contain end of input anchor '$' - See chevrotain.io/docs/guide/resolving_lexer_errors.html#ANCHORS for details.`,type:ir.LexerDefinitionErrorType.EOI_ANCHOR_FOUND,tokenTypes:[n]}});return i}Ve.findEndOfInputAnchor=EY;function IY(r){var e=(0,xe.filter)(r,function(i){var n=i[So];return n.test("")}),t=(0,xe.map)(e,function(i){return{message:"Token Type: ->"+i.name+"<- static 'PATTERN' must not match an empty string",type:ir.LexerDefinitionErrorType.EMPTY_MATCH_PATTERN,tokenTypes:[i]}});return t}Ve.findEmptyMatchRegExps=IY;var Yme=/[^\\[][\^]|^\^/;function yY(r){var e=function(n){hY(s,n);function s(){var o=n!==null&&n.apply(this,arguments)||this;return o.found=!1,o}return s.prototype.visitStartAnchor=function(o){this.found=!0},s}(pY.BaseRegExpVisitor),t=(0,xe.filter)(r,function(n){var s=n[So];try{var o=(0,dY.getRegExpAst)(s),a=new e;return a.visit(o),a.found}catch{return Yme.test(s.source)}}),i=(0,xe.map)(t,function(n){return{message:`Unexpected RegExp Anchor Error: - Token Type: ->`+n.name+`<- static 'PATTERN' cannot contain start of input anchor '^' - See https://chevrotain.io/docs/guide/resolving_lexer_errors.html#ANCHORS for details.`,type:ir.LexerDefinitionErrorType.SOI_ANCHOR_FOUND,tokenTypes:[n]}});return i}Ve.findStartOfInputAnchor=yY;function wY(r){var e=(0,xe.filter)(r,function(i){var n=i[So];return n instanceof RegExp&&(n.multiline||n.global)}),t=(0,xe.map)(e,function(i){return{message:"Token Type: ->"+i.name+"<- static 'PATTERN' may NOT contain global('g') or multiline('m')",type:ir.LexerDefinitionErrorType.UNSUPPORTED_FLAGS_FOUND,tokenTypes:[i]}});return t}Ve.findUnsupportedFlags=wY;function BY(r){var e=[],t=(0,xe.map)(r,function(s){return(0,xe.reduce)(r,function(o,a){return s.PATTERN.source===a.PATTERN.source&&!(0,xe.contains)(e,a)&&a.PATTERN!==ir.Lexer.NA&&(e.push(a),o.push(a)),o},[])});t=(0,xe.compact)(t);var i=(0,xe.filter)(t,function(s){return s.length>1}),n=(0,xe.map)(i,function(s){var o=(0,xe.map)(s,function(l){return l.name}),a=(0,xe.first)(s).PATTERN;return{message:"The same RegExp pattern ->"+a+"<-"+("has been used in all of the following Token Types: "+o.join(", ")+" <-"),type:ir.LexerDefinitionErrorType.DUPLICATE_PATTERNS_FOUND,tokenTypes:s}});return n}Ve.findDuplicatePatterns=BY;function QY(r){var e=(0,xe.filter)(r,function(i){if(!(0,xe.has)(i,"GROUP"))return!1;var n=i.GROUP;return n!==ir.Lexer.SKIPPED&&n!==ir.Lexer.NA&&!(0,xe.isString)(n)}),t=(0,xe.map)(e,function(i){return{message:"Token Type: ->"+i.name+"<- static 'GROUP' can only be Lexer.SKIPPED/Lexer.NA/A String",type:ir.LexerDefinitionErrorType.INVALID_GROUP_TYPE_FOUND,tokenTypes:[i]}});return t}Ve.findInvalidGroupType=QY;function bY(r,e){var t=(0,xe.filter)(r,function(n){return n.PUSH_MODE!==void 0&&!(0,xe.contains)(e,n.PUSH_MODE)}),i=(0,xe.map)(t,function(n){var s="Token Type: ->"+n.name+"<- static 'PUSH_MODE' value cannot refer to a Lexer Mode ->"+n.PUSH_MODE+"<-which does not exist";return{message:s,type:ir.LexerDefinitionErrorType.PUSH_MODE_DOES_NOT_EXIST,tokenTypes:[n]}});return i}Ve.findModesThatDoNotExist=bY;function SY(r){var e=[],t=(0,xe.reduce)(r,function(i,n,s){var o=n.PATTERN;return o===ir.Lexer.NA||((0,xe.isString)(o)?i.push({str:o,idx:s,tokenType:n}):(0,xe.isRegExp)(o)&&qme(o)&&i.push({str:o.source,idx:s,tokenType:n})),i},[]);return(0,xe.forEach)(r,function(i,n){(0,xe.forEach)(t,function(s){var o=s.str,a=s.idx,l=s.tokenType;if(n"+i.name+"<-")+`in the lexer's definition. -See https://chevrotain.io/docs/guide/resolving_lexer_errors.html#UNREACHABLE`;e.push({message:c,type:ir.LexerDefinitionErrorType.UNREACHABLE_PATTERN,tokenTypes:[i,l]})}})}),e}Ve.findUnreachablePatterns=SY;function jme(r,e){if((0,xe.isRegExp)(e)){var t=e.exec(r);return t!==null&&t.index===0}else{if((0,xe.isFunction)(e))return e(r,0,[],{});if((0,xe.has)(e,"exec"))return e.exec(r,0,[],{});if(typeof e=="string")return e===r;throw Error("non exhaustive match")}}function qme(r){var e=[".","\\","[","]","|","^","$","(",")","?","*","+","{"];return(0,xe.find)(e,function(t){return r.source.indexOf(t)!==-1})===void 0}function Ev(r){var e=r.ignoreCase?"i":"";return new RegExp("^(?:"+r.source+")",e)}Ve.addStartOfInput=Ev;function Iv(r){var e=r.ignoreCase?"iy":"y";return new RegExp(""+r.source,e)}Ve.addStickyFlag=Iv;function Jme(r,e,t){var i=[];return(0,xe.has)(r,Ve.DEFAULT_MODE)||i.push({message:"A MultiMode Lexer cannot be initialized without a <"+Ve.DEFAULT_MODE+`> property in its definition -`,type:ir.LexerDefinitionErrorType.MULTI_MODE_LEXER_WITHOUT_DEFAULT_MODE}),(0,xe.has)(r,Ve.MODES)||i.push({message:"A MultiMode Lexer cannot be initialized without a <"+Ve.MODES+`> property in its definition -`,type:ir.LexerDefinitionErrorType.MULTI_MODE_LEXER_WITHOUT_MODES_PROPERTY}),(0,xe.has)(r,Ve.MODES)&&(0,xe.has)(r,Ve.DEFAULT_MODE)&&!(0,xe.has)(r.modes,r.defaultMode)&&i.push({message:"A MultiMode Lexer cannot be initialized with a "+Ve.DEFAULT_MODE+": <"+r.defaultMode+`>which does not exist -`,type:ir.LexerDefinitionErrorType.MULTI_MODE_LEXER_DEFAULT_MODE_VALUE_DOES_NOT_EXIST}),(0,xe.has)(r,Ve.MODES)&&(0,xe.forEach)(r.modes,function(n,s){(0,xe.forEach)(n,function(o,a){(0,xe.isUndefined)(o)&&i.push({message:"A Lexer cannot be initialized using an undefined Token Type. Mode:"+("<"+s+"> at index: <"+a+`> -`),type:ir.LexerDefinitionErrorType.LEXER_DEFINITION_CANNOT_CONTAIN_UNDEFINED})})}),i}Ve.performRuntimeChecks=Jme;function Wme(r,e,t){var i=[],n=!1,s=(0,xe.compact)((0,xe.flatten)((0,xe.mapValues)(r.modes,function(l){return l}))),o=(0,xe.reject)(s,function(l){return l[So]===ir.Lexer.NA}),a=DY(t);return e&&(0,xe.forEach)(o,function(l){var c=xY(l,a);if(c!==!1){var u=PY(l,c),g={message:u,type:c.issue,tokenType:l};i.push(g)}else(0,xe.has)(l,"LINE_BREAKS")?l.LINE_BREAKS===!0&&(n=!0):(0,Hg.canMatchCharCode)(a,l.PATTERN)&&(n=!0)}),e&&!n&&i.push({message:`Warning: No LINE_BREAKS Found. - This Lexer has been defined to track line and column information, - But none of the Token Types can be identified as matching a line terminator. - See https://chevrotain.io/docs/guide/resolving_lexer_errors.html#LINE_BREAKS - for details.`,type:ir.LexerDefinitionErrorType.NO_LINE_BREAKS_FLAGS}),i}Ve.performWarningRuntimeChecks=Wme;function zme(r){var e={},t=(0,xe.keys)(r);return(0,xe.forEach)(t,function(i){var n=r[i];if((0,xe.isArray)(n))e[i]=[];else throw Error("non exhaustive match")}),e}Ve.cloneEmptyGroups=zme;function wv(r){var e=r.PATTERN;if((0,xe.isRegExp)(e))return!1;if((0,xe.isFunction)(e))return!0;if((0,xe.has)(e,"exec"))return!0;if((0,xe.isString)(e))return!1;throw Error("non exhaustive match")}Ve.isCustomPattern=wv;function vY(r){return(0,xe.isString)(r)&&r.length===1?r.charCodeAt(0):!1}Ve.isShortPattern=vY;Ve.LineTerminatorOptimizedTester={test:function(r){for(var e=r.length,t=this.lastIndex;t Token Type -`)+(" Root cause: "+e.errMsg+`. -`)+" For details See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#IDENTIFY_TERMINATOR";if(e.issue===ir.LexerDefinitionErrorType.CUSTOM_LINE_BREAK)return`Warning: A Custom Token Pattern should specify the option. -`+(" The problem is in the <"+r.name+`> Token Type -`)+" For details See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#CUSTOM_LINE_BREAK";throw Error("non exhaustive match")}Ve.buildLineBreakIssueMessage=PY;function DY(r){var e=(0,xe.map)(r,function(t){return(0,xe.isString)(t)&&t.length>0?t.charCodeAt(0):t});return e}function mv(r,e,t){r[e]===void 0?r[e]=[t]:r[e].push(t)}Ve.minOptimizationVal=256;var qI=[];function yv(r){return r255?255+~~(r/255):r}}});var Gg=w(Nt=>{"use strict";Object.defineProperty(Nt,"__esModule",{value:!0});Nt.isTokenType=Nt.hasExtendingTokensTypesMapProperty=Nt.hasExtendingTokensTypesProperty=Nt.hasCategoriesProperty=Nt.hasShortKeyProperty=Nt.singleAssignCategoriesToksMap=Nt.assignCategoriesMapProp=Nt.assignCategoriesTokensProp=Nt.assignTokenDefaultProps=Nt.expandCategories=Nt.augmentTokenTypes=Nt.tokenIdxToClass=Nt.tokenShortNameIdx=Nt.tokenStructuredMatcherNoCategories=Nt.tokenStructuredMatcher=void 0;var _r=Gt();function Xme(r,e){var t=r.tokenTypeIdx;return t===e.tokenTypeIdx?!0:e.isParent===!0&&e.categoryMatchesMap[t]===!0}Nt.tokenStructuredMatcher=Xme;function _me(r,e){return r.tokenTypeIdx===e.tokenTypeIdx}Nt.tokenStructuredMatcherNoCategories=_me;Nt.tokenShortNameIdx=1;Nt.tokenIdxToClass={};function Zme(r){var e=kY(r);RY(e),NY(e),FY(e),(0,_r.forEach)(e,function(t){t.isParent=t.categoryMatches.length>0})}Nt.augmentTokenTypes=Zme;function kY(r){for(var e=(0,_r.cloneArr)(r),t=r,i=!0;i;){t=(0,_r.compact)((0,_r.flatten)((0,_r.map)(t,function(s){return s.CATEGORIES})));var n=(0,_r.difference)(t,e);e=e.concat(n),(0,_r.isEmpty)(n)?i=!1:t=n}return e}Nt.expandCategories=kY;function RY(r){(0,_r.forEach)(r,function(e){LY(e)||(Nt.tokenIdxToClass[Nt.tokenShortNameIdx]=e,e.tokenTypeIdx=Nt.tokenShortNameIdx++),Bv(e)&&!(0,_r.isArray)(e.CATEGORIES)&&(e.CATEGORIES=[e.CATEGORIES]),Bv(e)||(e.CATEGORIES=[]),TY(e)||(e.categoryMatches=[]),OY(e)||(e.categoryMatchesMap={})})}Nt.assignTokenDefaultProps=RY;function FY(r){(0,_r.forEach)(r,function(e){e.categoryMatches=[],(0,_r.forEach)(e.categoryMatchesMap,function(t,i){e.categoryMatches.push(Nt.tokenIdxToClass[i].tokenTypeIdx)})})}Nt.assignCategoriesTokensProp=FY;function NY(r){(0,_r.forEach)(r,function(e){Qv([],e)})}Nt.assignCategoriesMapProp=NY;function Qv(r,e){(0,_r.forEach)(r,function(t){e.categoryMatchesMap[t.tokenTypeIdx]=!0}),(0,_r.forEach)(e.CATEGORIES,function(t){var i=r.concat(e);(0,_r.contains)(i,t)||Qv(i,t)})}Nt.singleAssignCategoriesToksMap=Qv;function LY(r){return(0,_r.has)(r,"tokenTypeIdx")}Nt.hasShortKeyProperty=LY;function Bv(r){return(0,_r.has)(r,"CATEGORIES")}Nt.hasCategoriesProperty=Bv;function TY(r){return(0,_r.has)(r,"categoryMatches")}Nt.hasExtendingTokensTypesProperty=TY;function OY(r){return(0,_r.has)(r,"categoryMatchesMap")}Nt.hasExtendingTokensTypesMapProperty=OY;function $me(r){return(0,_r.has)(r,"tokenTypeIdx")}Nt.isTokenType=$me});var bv=w(JI=>{"use strict";Object.defineProperty(JI,"__esModule",{value:!0});JI.defaultLexerErrorProvider=void 0;JI.defaultLexerErrorProvider={buildUnableToPopLexerModeMessage:function(r){return"Unable to pop Lexer Mode after encountering Token ->"+r.image+"<- The Mode Stack is empty"},buildUnexpectedCharactersMessage:function(r,e,t,i,n){return"unexpected character: ->"+r.charAt(e)+"<- at offset: "+e+","+(" skipped "+t+" characters.")}}});var gd=w(nc=>{"use strict";Object.defineProperty(nc,"__esModule",{value:!0});nc.Lexer=nc.LexerDefinitionErrorType=void 0;var zs=Cv(),nr=Gt(),eEe=Gg(),tEe=bv(),rEe=GI(),iEe;(function(r){r[r.MISSING_PATTERN=0]="MISSING_PATTERN",r[r.INVALID_PATTERN=1]="INVALID_PATTERN",r[r.EOI_ANCHOR_FOUND=2]="EOI_ANCHOR_FOUND",r[r.UNSUPPORTED_FLAGS_FOUND=3]="UNSUPPORTED_FLAGS_FOUND",r[r.DUPLICATE_PATTERNS_FOUND=4]="DUPLICATE_PATTERNS_FOUND",r[r.INVALID_GROUP_TYPE_FOUND=5]="INVALID_GROUP_TYPE_FOUND",r[r.PUSH_MODE_DOES_NOT_EXIST=6]="PUSH_MODE_DOES_NOT_EXIST",r[r.MULTI_MODE_LEXER_WITHOUT_DEFAULT_MODE=7]="MULTI_MODE_LEXER_WITHOUT_DEFAULT_MODE",r[r.MULTI_MODE_LEXER_WITHOUT_MODES_PROPERTY=8]="MULTI_MODE_LEXER_WITHOUT_MODES_PROPERTY",r[r.MULTI_MODE_LEXER_DEFAULT_MODE_VALUE_DOES_NOT_EXIST=9]="MULTI_MODE_LEXER_DEFAULT_MODE_VALUE_DOES_NOT_EXIST",r[r.LEXER_DEFINITION_CANNOT_CONTAIN_UNDEFINED=10]="LEXER_DEFINITION_CANNOT_CONTAIN_UNDEFINED",r[r.SOI_ANCHOR_FOUND=11]="SOI_ANCHOR_FOUND",r[r.EMPTY_MATCH_PATTERN=12]="EMPTY_MATCH_PATTERN",r[r.NO_LINE_BREAKS_FLAGS=13]="NO_LINE_BREAKS_FLAGS",r[r.UNREACHABLE_PATTERN=14]="UNREACHABLE_PATTERN",r[r.IDENTIFY_TERMINATOR=15]="IDENTIFY_TERMINATOR",r[r.CUSTOM_LINE_BREAK=16]="CUSTOM_LINE_BREAK"})(iEe=nc.LexerDefinitionErrorType||(nc.LexerDefinitionErrorType={}));var fd={deferDefinitionErrorsHandling:!1,positionTracking:"full",lineTerminatorsPattern:/\n|\r\n?/g,lineTerminatorCharacters:[` -`,"\r"],ensureOptimizations:!1,safeMode:!1,errorMessageProvider:tEe.defaultLexerErrorProvider,traceInitPerf:!1,skipValidations:!1};Object.freeze(fd);var nEe=function(){function r(e,t){var i=this;if(t===void 0&&(t=fd),this.lexerDefinition=e,this.lexerDefinitionErrors=[],this.lexerDefinitionWarning=[],this.patternIdxToConfig={},this.charCodeToPatternIdxToConfig={},this.modes=[],this.emptyGroups={},this.config=void 0,this.trackStartLines=!0,this.trackEndLines=!0,this.hasCustom=!1,this.canModeBeOptimized={},typeof t=="boolean")throw Error(`The second argument to the Lexer constructor is now an ILexerConfig Object. -a boolean 2nd argument is no longer supported`);this.config=(0,nr.merge)(fd,t);var n=this.config.traceInitPerf;n===!0?(this.traceInitMaxIdent=1/0,this.traceInitPerf=!0):typeof n=="number"&&(this.traceInitMaxIdent=n,this.traceInitPerf=!0),this.traceInitIndent=-1,this.TRACE_INIT("Lexer Constructor",function(){var s,o=!0;i.TRACE_INIT("Lexer Config handling",function(){if(i.config.lineTerminatorsPattern===fd.lineTerminatorsPattern)i.config.lineTerminatorsPattern=zs.LineTerminatorOptimizedTester;else if(i.config.lineTerminatorCharacters===fd.lineTerminatorCharacters)throw Error(`Error: Missing property on the Lexer config. - For details See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#MISSING_LINE_TERM_CHARS`);if(t.safeMode&&t.ensureOptimizations)throw Error('"safeMode" and "ensureOptimizations" flags are mutually exclusive.');i.trackStartLines=/full|onlyStart/i.test(i.config.positionTracking),i.trackEndLines=/full/i.test(i.config.positionTracking),(0,nr.isArray)(e)?(s={modes:{}},s.modes[zs.DEFAULT_MODE]=(0,nr.cloneArr)(e),s[zs.DEFAULT_MODE]=zs.DEFAULT_MODE):(o=!1,s=(0,nr.cloneObj)(e))}),i.config.skipValidations===!1&&(i.TRACE_INIT("performRuntimeChecks",function(){i.lexerDefinitionErrors=i.lexerDefinitionErrors.concat((0,zs.performRuntimeChecks)(s,i.trackStartLines,i.config.lineTerminatorCharacters))}),i.TRACE_INIT("performWarningRuntimeChecks",function(){i.lexerDefinitionWarning=i.lexerDefinitionWarning.concat((0,zs.performWarningRuntimeChecks)(s,i.trackStartLines,i.config.lineTerminatorCharacters))})),s.modes=s.modes?s.modes:{},(0,nr.forEach)(s.modes,function(u,g){s.modes[g]=(0,nr.reject)(u,function(f){return(0,nr.isUndefined)(f)})});var a=(0,nr.keys)(s.modes);if((0,nr.forEach)(s.modes,function(u,g){i.TRACE_INIT("Mode: <"+g+"> processing",function(){if(i.modes.push(g),i.config.skipValidations===!1&&i.TRACE_INIT("validatePatterns",function(){i.lexerDefinitionErrors=i.lexerDefinitionErrors.concat((0,zs.validatePatterns)(u,a))}),(0,nr.isEmpty)(i.lexerDefinitionErrors)){(0,eEe.augmentTokenTypes)(u);var f;i.TRACE_INIT("analyzeTokenTypes",function(){f=(0,zs.analyzeTokenTypes)(u,{lineTerminatorCharacters:i.config.lineTerminatorCharacters,positionTracking:t.positionTracking,ensureOptimizations:t.ensureOptimizations,safeMode:t.safeMode,tracer:i.TRACE_INIT.bind(i)})}),i.patternIdxToConfig[g]=f.patternIdxToConfig,i.charCodeToPatternIdxToConfig[g]=f.charCodeToPatternIdxToConfig,i.emptyGroups=(0,nr.merge)(i.emptyGroups,f.emptyGroups),i.hasCustom=f.hasCustom||i.hasCustom,i.canModeBeOptimized[g]=f.canBeOptimized}})}),i.defaultMode=s.defaultMode,!(0,nr.isEmpty)(i.lexerDefinitionErrors)&&!i.config.deferDefinitionErrorsHandling){var l=(0,nr.map)(i.lexerDefinitionErrors,function(u){return u.message}),c=l.join(`----------------------- -`);throw new Error(`Errors detected in definition of Lexer: -`+c)}(0,nr.forEach)(i.lexerDefinitionWarning,function(u){(0,nr.PRINT_WARNING)(u.message)}),i.TRACE_INIT("Choosing sub-methods implementations",function(){if(zs.SUPPORT_STICKY?(i.chopInput=nr.IDENTITY,i.match=i.matchWithTest):(i.updateLastIndex=nr.NOOP,i.match=i.matchWithExec),o&&(i.handleModes=nr.NOOP),i.trackStartLines===!1&&(i.computeNewColumn=nr.IDENTITY),i.trackEndLines===!1&&(i.updateTokenEndLineColumnLocation=nr.NOOP),/full/i.test(i.config.positionTracking))i.createTokenInstance=i.createFullToken;else if(/onlyStart/i.test(i.config.positionTracking))i.createTokenInstance=i.createStartOnlyToken;else if(/onlyOffset/i.test(i.config.positionTracking))i.createTokenInstance=i.createOffsetOnlyToken;else throw Error('Invalid config option: "'+i.config.positionTracking+'"');i.hasCustom?(i.addToken=i.addTokenUsingPush,i.handlePayload=i.handlePayloadWithCustom):(i.addToken=i.addTokenUsingMemberAccess,i.handlePayload=i.handlePayloadNoCustom)}),i.TRACE_INIT("Failed Optimization Warnings",function(){var u=(0,nr.reduce)(i.canModeBeOptimized,function(g,f,h){return f===!1&&g.push(h),g},[]);if(t.ensureOptimizations&&!(0,nr.isEmpty)(u))throw Error("Lexer Modes: < "+u.join(", ")+` > cannot be optimized. - Disable the "ensureOptimizations" lexer config flag to silently ignore this and run the lexer in an un-optimized mode. - Or inspect the console log for details on how to resolve these issues.`)}),i.TRACE_INIT("clearRegExpParserCache",function(){(0,rEe.clearRegExpParserCache)()}),i.TRACE_INIT("toFastProperties",function(){(0,nr.toFastProperties)(i)})})}return r.prototype.tokenize=function(e,t){if(t===void 0&&(t=this.defaultMode),!(0,nr.isEmpty)(this.lexerDefinitionErrors)){var i=(0,nr.map)(this.lexerDefinitionErrors,function(o){return o.message}),n=i.join(`----------------------- -`);throw new Error(`Unable to Tokenize because Errors detected in definition of Lexer: -`+n)}var s=this.tokenizeInternal(e,t);return s},r.prototype.tokenizeInternal=function(e,t){var i=this,n,s,o,a,l,c,u,g,f,h,p,C,y,B,v,D,L=e,H=L.length,j=0,$=0,V=this.hasCustom?0:Math.floor(e.length/10),W=new Array(V),Z=[],A=this.trackStartLines?1:void 0,ae=this.trackStartLines?1:void 0,ge=(0,zs.cloneEmptyGroups)(this.emptyGroups),re=this.trackStartLines,O=this.config.lineTerminatorsPattern,F=0,ue=[],he=[],ke=[],Fe=[];Object.freeze(Fe);var Ne=void 0;function oe(){return ue}function le(pr){var Ei=(0,zs.charCodeToOptimizedIndex)(pr),_n=he[Ei];return _n===void 0?Fe:_n}var we=function(pr){if(ke.length===1&&pr.tokenType.PUSH_MODE===void 0){var Ei=i.config.errorMessageProvider.buildUnableToPopLexerModeMessage(pr);Z.push({offset:pr.startOffset,line:pr.startLine!==void 0?pr.startLine:void 0,column:pr.startColumn!==void 0?pr.startColumn:void 0,length:pr.image.length,message:Ei})}else{ke.pop();var _n=(0,nr.last)(ke);ue=i.patternIdxToConfig[_n],he=i.charCodeToPatternIdxToConfig[_n],F=ue.length;var oa=i.canModeBeOptimized[_n]&&i.config.safeMode===!1;he&&oa?Ne=le:Ne=oe}};function fe(pr){ke.push(pr),he=this.charCodeToPatternIdxToConfig[pr],ue=this.patternIdxToConfig[pr],F=ue.length,F=ue.length;var Ei=this.canModeBeOptimized[pr]&&this.config.safeMode===!1;he&&Ei?Ne=le:Ne=oe}fe.call(this,t);for(var Ae;jc.length){c=a,u=g,Ae=tt;break}}}break}}if(c!==null){if(f=c.length,h=Ae.group,h!==void 0&&(p=Ae.tokenTypeIdx,C=this.createTokenInstance(c,j,p,Ae.tokenType,A,ae,f),this.handlePayload(C,u),h===!1?$=this.addToken(W,$,C):ge[h].push(C)),e=this.chopInput(e,f),j=j+f,ae=this.computeNewColumn(ae,f),re===!0&&Ae.canLineTerminator===!0){var It=0,Or=void 0,ii=void 0;O.lastIndex=0;do Or=O.test(c),Or===!0&&(ii=O.lastIndex-1,It++);while(Or===!0);It!==0&&(A=A+It,ae=f-ii,this.updateTokenEndLineColumnLocation(C,h,ii,It,A,ae,f))}this.handleModes(Ae,we,fe,C)}else{for(var gi=j,hr=A,fi=ae,ni=!1;!ni&&j <"+e+">");var n=(0,nr.timer)(t),s=n.time,o=n.value,a=s>10?console.warn:console.log;return this.traceInitIndent time: "+s+"ms"),this.traceInitIndent--,o}else return t()},r.SKIPPED="This marks a skipped Token pattern, this means each token identified by it willbe consumed and then thrown into oblivion, this can be used to for example to completely ignore whitespace.",r.NA=/NOT_APPLICABLE/,r}();nc.Lexer=nEe});var SA=w(Qi=>{"use strict";Object.defineProperty(Qi,"__esModule",{value:!0});Qi.tokenMatcher=Qi.createTokenInstance=Qi.EOF=Qi.createToken=Qi.hasTokenLabel=Qi.tokenName=Qi.tokenLabel=void 0;var Vs=Gt(),sEe=gd(),Sv=Gg();function oEe(r){return JY(r)?r.LABEL:r.name}Qi.tokenLabel=oEe;function aEe(r){return r.name}Qi.tokenName=aEe;function JY(r){return(0,Vs.isString)(r.LABEL)&&r.LABEL!==""}Qi.hasTokenLabel=JY;var AEe="parent",MY="categories",KY="label",UY="group",HY="push_mode",GY="pop_mode",YY="longer_alt",jY="line_breaks",qY="start_chars_hint";function WY(r){return lEe(r)}Qi.createToken=WY;function lEe(r){var e=r.pattern,t={};if(t.name=r.name,(0,Vs.isUndefined)(e)||(t.PATTERN=e),(0,Vs.has)(r,AEe))throw`The parent property is no longer supported. -See: https://github.com/chevrotain/chevrotain/issues/564#issuecomment-349062346 for details.`;return(0,Vs.has)(r,MY)&&(t.CATEGORIES=r[MY]),(0,Sv.augmentTokenTypes)([t]),(0,Vs.has)(r,KY)&&(t.LABEL=r[KY]),(0,Vs.has)(r,UY)&&(t.GROUP=r[UY]),(0,Vs.has)(r,GY)&&(t.POP_MODE=r[GY]),(0,Vs.has)(r,HY)&&(t.PUSH_MODE=r[HY]),(0,Vs.has)(r,YY)&&(t.LONGER_ALT=r[YY]),(0,Vs.has)(r,jY)&&(t.LINE_BREAKS=r[jY]),(0,Vs.has)(r,qY)&&(t.START_CHARS_HINT=r[qY]),t}Qi.EOF=WY({name:"EOF",pattern:sEe.Lexer.NA});(0,Sv.augmentTokenTypes)([Qi.EOF]);function cEe(r,e,t,i,n,s,o,a){return{image:e,startOffset:t,endOffset:i,startLine:n,endLine:s,startColumn:o,endColumn:a,tokenTypeIdx:r.tokenTypeIdx,tokenType:r}}Qi.createTokenInstance=cEe;function uEe(r,e){return(0,Sv.tokenStructuredMatcher)(r,e)}Qi.tokenMatcher=uEe});var dn=w(Wt=>{"use strict";var wa=Wt&&Wt.__extends||function(){var r=function(e,t){return r=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(i,n){i.__proto__=n}||function(i,n){for(var s in n)Object.prototype.hasOwnProperty.call(n,s)&&(i[s]=n[s])},r(e,t)};return function(e,t){if(typeof t!="function"&&t!==null)throw new TypeError("Class extends value "+String(t)+" is not a constructor or null");r(e,t);function i(){this.constructor=e}e.prototype=t===null?Object.create(t):(i.prototype=t.prototype,new i)}}();Object.defineProperty(Wt,"__esModule",{value:!0});Wt.serializeProduction=Wt.serializeGrammar=Wt.Terminal=Wt.Alternation=Wt.RepetitionWithSeparator=Wt.Repetition=Wt.RepetitionMandatoryWithSeparator=Wt.RepetitionMandatory=Wt.Option=Wt.Alternative=Wt.Rule=Wt.NonTerminal=Wt.AbstractProduction=void 0;var Ar=Gt(),gEe=SA(),vo=function(){function r(e){this._definition=e}return Object.defineProperty(r.prototype,"definition",{get:function(){return this._definition},set:function(e){this._definition=e},enumerable:!1,configurable:!0}),r.prototype.accept=function(e){e.visit(this),(0,Ar.forEach)(this.definition,function(t){t.accept(e)})},r}();Wt.AbstractProduction=vo;var zY=function(r){wa(e,r);function e(t){var i=r.call(this,[])||this;return i.idx=1,(0,Ar.assign)(i,(0,Ar.pick)(t,function(n){return n!==void 0})),i}return Object.defineProperty(e.prototype,"definition",{get:function(){return this.referencedRule!==void 0?this.referencedRule.definition:[]},set:function(t){},enumerable:!1,configurable:!0}),e.prototype.accept=function(t){t.visit(this)},e}(vo);Wt.NonTerminal=zY;var VY=function(r){wa(e,r);function e(t){var i=r.call(this,t.definition)||this;return i.orgText="",(0,Ar.assign)(i,(0,Ar.pick)(t,function(n){return n!==void 0})),i}return e}(vo);Wt.Rule=VY;var XY=function(r){wa(e,r);function e(t){var i=r.call(this,t.definition)||this;return i.ignoreAmbiguities=!1,(0,Ar.assign)(i,(0,Ar.pick)(t,function(n){return n!==void 0})),i}return e}(vo);Wt.Alternative=XY;var _Y=function(r){wa(e,r);function e(t){var i=r.call(this,t.definition)||this;return i.idx=1,(0,Ar.assign)(i,(0,Ar.pick)(t,function(n){return n!==void 0})),i}return e}(vo);Wt.Option=_Y;var ZY=function(r){wa(e,r);function e(t){var i=r.call(this,t.definition)||this;return i.idx=1,(0,Ar.assign)(i,(0,Ar.pick)(t,function(n){return n!==void 0})),i}return e}(vo);Wt.RepetitionMandatory=ZY;var $Y=function(r){wa(e,r);function e(t){var i=r.call(this,t.definition)||this;return i.idx=1,(0,Ar.assign)(i,(0,Ar.pick)(t,function(n){return n!==void 0})),i}return e}(vo);Wt.RepetitionMandatoryWithSeparator=$Y;var ej=function(r){wa(e,r);function e(t){var i=r.call(this,t.definition)||this;return i.idx=1,(0,Ar.assign)(i,(0,Ar.pick)(t,function(n){return n!==void 0})),i}return e}(vo);Wt.Repetition=ej;var tj=function(r){wa(e,r);function e(t){var i=r.call(this,t.definition)||this;return i.idx=1,(0,Ar.assign)(i,(0,Ar.pick)(t,function(n){return n!==void 0})),i}return e}(vo);Wt.RepetitionWithSeparator=tj;var rj=function(r){wa(e,r);function e(t){var i=r.call(this,t.definition)||this;return i.idx=1,i.ignoreAmbiguities=!1,i.hasPredicates=!1,(0,Ar.assign)(i,(0,Ar.pick)(t,function(n){return n!==void 0})),i}return Object.defineProperty(e.prototype,"definition",{get:function(){return this._definition},set:function(t){this._definition=t},enumerable:!1,configurable:!0}),e}(vo);Wt.Alternation=rj;var WI=function(){function r(e){this.idx=1,(0,Ar.assign)(this,(0,Ar.pick)(e,function(t){return t!==void 0}))}return r.prototype.accept=function(e){e.visit(this)},r}();Wt.Terminal=WI;function fEe(r){return(0,Ar.map)(r,hd)}Wt.serializeGrammar=fEe;function hd(r){function e(s){return(0,Ar.map)(s,hd)}if(r instanceof zY){var t={type:"NonTerminal",name:r.nonTerminalName,idx:r.idx};return(0,Ar.isString)(r.label)&&(t.label=r.label),t}else{if(r instanceof XY)return{type:"Alternative",definition:e(r.definition)};if(r instanceof _Y)return{type:"Option",idx:r.idx,definition:e(r.definition)};if(r instanceof ZY)return{type:"RepetitionMandatory",idx:r.idx,definition:e(r.definition)};if(r instanceof $Y)return{type:"RepetitionMandatoryWithSeparator",idx:r.idx,separator:hd(new WI({terminalType:r.separator})),definition:e(r.definition)};if(r instanceof tj)return{type:"RepetitionWithSeparator",idx:r.idx,separator:hd(new WI({terminalType:r.separator})),definition:e(r.definition)};if(r instanceof ej)return{type:"Repetition",idx:r.idx,definition:e(r.definition)};if(r instanceof rj)return{type:"Alternation",idx:r.idx,definition:e(r.definition)};if(r instanceof WI){var i={type:"Terminal",name:r.terminalType.name,label:(0,gEe.tokenLabel)(r.terminalType),idx:r.idx};(0,Ar.isString)(r.label)&&(i.terminalLabel=r.label);var n=r.terminalType.PATTERN;return r.terminalType.PATTERN&&(i.pattern=(0,Ar.isRegExp)(n)?n.source:n),i}else{if(r instanceof VY)return{type:"Rule",name:r.name,orgText:r.orgText,definition:e(r.definition)};throw Error("non exhaustive match")}}}Wt.serializeProduction=hd});var VI=w(zI=>{"use strict";Object.defineProperty(zI,"__esModule",{value:!0});zI.RestWalker=void 0;var vv=Gt(),Cn=dn(),hEe=function(){function r(){}return r.prototype.walk=function(e,t){var i=this;t===void 0&&(t=[]),(0,vv.forEach)(e.definition,function(n,s){var o=(0,vv.drop)(e.definition,s+1);if(n instanceof Cn.NonTerminal)i.walkProdRef(n,o,t);else if(n instanceof Cn.Terminal)i.walkTerminal(n,o,t);else if(n instanceof Cn.Alternative)i.walkFlat(n,o,t);else if(n instanceof Cn.Option)i.walkOption(n,o,t);else if(n instanceof Cn.RepetitionMandatory)i.walkAtLeastOne(n,o,t);else if(n instanceof Cn.RepetitionMandatoryWithSeparator)i.walkAtLeastOneSep(n,o,t);else if(n instanceof Cn.RepetitionWithSeparator)i.walkManySep(n,o,t);else if(n instanceof Cn.Repetition)i.walkMany(n,o,t);else if(n instanceof Cn.Alternation)i.walkOr(n,o,t);else throw Error("non exhaustive match")})},r.prototype.walkTerminal=function(e,t,i){},r.prototype.walkProdRef=function(e,t,i){},r.prototype.walkFlat=function(e,t,i){var n=t.concat(i);this.walk(e,n)},r.prototype.walkOption=function(e,t,i){var n=t.concat(i);this.walk(e,n)},r.prototype.walkAtLeastOne=function(e,t,i){var n=[new Cn.Option({definition:e.definition})].concat(t,i);this.walk(e,n)},r.prototype.walkAtLeastOneSep=function(e,t,i){var n=ij(e,t,i);this.walk(e,n)},r.prototype.walkMany=function(e,t,i){var n=[new Cn.Option({definition:e.definition})].concat(t,i);this.walk(e,n)},r.prototype.walkManySep=function(e,t,i){var n=ij(e,t,i);this.walk(e,n)},r.prototype.walkOr=function(e,t,i){var n=this,s=t.concat(i);(0,vv.forEach)(e.definition,function(o){var a=new Cn.Alternative({definition:[o]});n.walk(a,s)})},r}();zI.RestWalker=hEe;function ij(r,e,t){var i=[new Cn.Option({definition:[new Cn.Terminal({terminalType:r.separator})].concat(r.definition)})],n=i.concat(e,t);return n}});var Yg=w(XI=>{"use strict";Object.defineProperty(XI,"__esModule",{value:!0});XI.GAstVisitor=void 0;var xo=dn(),pEe=function(){function r(){}return r.prototype.visit=function(e){var t=e;switch(t.constructor){case xo.NonTerminal:return this.visitNonTerminal(t);case xo.Alternative:return this.visitAlternative(t);case xo.Option:return this.visitOption(t);case xo.RepetitionMandatory:return this.visitRepetitionMandatory(t);case xo.RepetitionMandatoryWithSeparator:return this.visitRepetitionMandatoryWithSeparator(t);case xo.RepetitionWithSeparator:return this.visitRepetitionWithSeparator(t);case xo.Repetition:return this.visitRepetition(t);case xo.Alternation:return this.visitAlternation(t);case xo.Terminal:return this.visitTerminal(t);case xo.Rule:return this.visitRule(t);default:throw Error("non exhaustive match")}},r.prototype.visitNonTerminal=function(e){},r.prototype.visitAlternative=function(e){},r.prototype.visitOption=function(e){},r.prototype.visitRepetition=function(e){},r.prototype.visitRepetitionMandatory=function(e){},r.prototype.visitRepetitionMandatoryWithSeparator=function(e){},r.prototype.visitRepetitionWithSeparator=function(e){},r.prototype.visitAlternation=function(e){},r.prototype.visitTerminal=function(e){},r.prototype.visitRule=function(e){},r}();XI.GAstVisitor=pEe});var dd=w(Oi=>{"use strict";var dEe=Oi&&Oi.__extends||function(){var r=function(e,t){return r=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(i,n){i.__proto__=n}||function(i,n){for(var s in n)Object.prototype.hasOwnProperty.call(n,s)&&(i[s]=n[s])},r(e,t)};return function(e,t){if(typeof t!="function"&&t!==null)throw new TypeError("Class extends value "+String(t)+" is not a constructor or null");r(e,t);function i(){this.constructor=e}e.prototype=t===null?Object.create(t):(i.prototype=t.prototype,new i)}}();Object.defineProperty(Oi,"__esModule",{value:!0});Oi.collectMethods=Oi.DslMethodsCollectorVisitor=Oi.getProductionDslName=Oi.isBranchingProd=Oi.isOptionalProd=Oi.isSequenceProd=void 0;var pd=Gt(),Qr=dn(),CEe=Yg();function mEe(r){return r instanceof Qr.Alternative||r instanceof Qr.Option||r instanceof Qr.Repetition||r instanceof Qr.RepetitionMandatory||r instanceof Qr.RepetitionMandatoryWithSeparator||r instanceof Qr.RepetitionWithSeparator||r instanceof Qr.Terminal||r instanceof Qr.Rule}Oi.isSequenceProd=mEe;function xv(r,e){e===void 0&&(e=[]);var t=r instanceof Qr.Option||r instanceof Qr.Repetition||r instanceof Qr.RepetitionWithSeparator;return t?!0:r instanceof Qr.Alternation?(0,pd.some)(r.definition,function(i){return xv(i,e)}):r instanceof Qr.NonTerminal&&(0,pd.contains)(e,r)?!1:r instanceof Qr.AbstractProduction?(r instanceof Qr.NonTerminal&&e.push(r),(0,pd.every)(r.definition,function(i){return xv(i,e)})):!1}Oi.isOptionalProd=xv;function EEe(r){return r instanceof Qr.Alternation}Oi.isBranchingProd=EEe;function IEe(r){if(r instanceof Qr.NonTerminal)return"SUBRULE";if(r instanceof Qr.Option)return"OPTION";if(r instanceof Qr.Alternation)return"OR";if(r instanceof Qr.RepetitionMandatory)return"AT_LEAST_ONE";if(r instanceof Qr.RepetitionMandatoryWithSeparator)return"AT_LEAST_ONE_SEP";if(r instanceof Qr.RepetitionWithSeparator)return"MANY_SEP";if(r instanceof Qr.Repetition)return"MANY";if(r instanceof Qr.Terminal)return"CONSUME";throw Error("non exhaustive match")}Oi.getProductionDslName=IEe;var nj=function(r){dEe(e,r);function e(){var t=r!==null&&r.apply(this,arguments)||this;return t.separator="-",t.dslMethods={option:[],alternation:[],repetition:[],repetitionWithSeparator:[],repetitionMandatory:[],repetitionMandatoryWithSeparator:[]},t}return e.prototype.reset=function(){this.dslMethods={option:[],alternation:[],repetition:[],repetitionWithSeparator:[],repetitionMandatory:[],repetitionMandatoryWithSeparator:[]}},e.prototype.visitTerminal=function(t){var i=t.terminalType.name+this.separator+"Terminal";(0,pd.has)(this.dslMethods,i)||(this.dslMethods[i]=[]),this.dslMethods[i].push(t)},e.prototype.visitNonTerminal=function(t){var i=t.nonTerminalName+this.separator+"Terminal";(0,pd.has)(this.dslMethods,i)||(this.dslMethods[i]=[]),this.dslMethods[i].push(t)},e.prototype.visitOption=function(t){this.dslMethods.option.push(t)},e.prototype.visitRepetitionWithSeparator=function(t){this.dslMethods.repetitionWithSeparator.push(t)},e.prototype.visitRepetitionMandatory=function(t){this.dslMethods.repetitionMandatory.push(t)},e.prototype.visitRepetitionMandatoryWithSeparator=function(t){this.dslMethods.repetitionMandatoryWithSeparator.push(t)},e.prototype.visitRepetition=function(t){this.dslMethods.repetition.push(t)},e.prototype.visitAlternation=function(t){this.dslMethods.alternation.push(t)},e}(CEe.GAstVisitor);Oi.DslMethodsCollectorVisitor=nj;var _I=new nj;function yEe(r){_I.reset(),r.accept(_I);var e=_I.dslMethods;return _I.reset(),e}Oi.collectMethods=yEe});var Dv=w(Po=>{"use strict";Object.defineProperty(Po,"__esModule",{value:!0});Po.firstForTerminal=Po.firstForBranching=Po.firstForSequence=Po.first=void 0;var ZI=Gt(),sj=dn(),Pv=dd();function $I(r){if(r instanceof sj.NonTerminal)return $I(r.referencedRule);if(r instanceof sj.Terminal)return Aj(r);if((0,Pv.isSequenceProd)(r))return oj(r);if((0,Pv.isBranchingProd)(r))return aj(r);throw Error("non exhaustive match")}Po.first=$I;function oj(r){for(var e=[],t=r.definition,i=0,n=t.length>i,s,o=!0;n&&o;)s=t[i],o=(0,Pv.isOptionalProd)(s),e=e.concat($I(s)),i=i+1,n=t.length>i;return(0,ZI.uniq)(e)}Po.firstForSequence=oj;function aj(r){var e=(0,ZI.map)(r.definition,function(t){return $I(t)});return(0,ZI.uniq)((0,ZI.flatten)(e))}Po.firstForBranching=aj;function Aj(r){return[r.terminalType]}Po.firstForTerminal=Aj});var kv=w(ey=>{"use strict";Object.defineProperty(ey,"__esModule",{value:!0});ey.IN=void 0;ey.IN="_~IN~_"});var fj=w(As=>{"use strict";var wEe=As&&As.__extends||function(){var r=function(e,t){return r=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(i,n){i.__proto__=n}||function(i,n){for(var s in n)Object.prototype.hasOwnProperty.call(n,s)&&(i[s]=n[s])},r(e,t)};return function(e,t){if(typeof t!="function"&&t!==null)throw new TypeError("Class extends value "+String(t)+" is not a constructor or null");r(e,t);function i(){this.constructor=e}e.prototype=t===null?Object.create(t):(i.prototype=t.prototype,new i)}}();Object.defineProperty(As,"__esModule",{value:!0});As.buildInProdFollowPrefix=As.buildBetweenProdsFollowPrefix=As.computeAllProdsFollows=As.ResyncFollowsWalker=void 0;var BEe=VI(),QEe=Dv(),lj=Gt(),cj=kv(),bEe=dn(),uj=function(r){wEe(e,r);function e(t){var i=r.call(this)||this;return i.topProd=t,i.follows={},i}return e.prototype.startWalking=function(){return this.walk(this.topProd),this.follows},e.prototype.walkTerminal=function(t,i,n){},e.prototype.walkProdRef=function(t,i,n){var s=gj(t.referencedRule,t.idx)+this.topProd.name,o=i.concat(n),a=new bEe.Alternative({definition:o}),l=(0,QEe.first)(a);this.follows[s]=l},e}(BEe.RestWalker);As.ResyncFollowsWalker=uj;function SEe(r){var e={};return(0,lj.forEach)(r,function(t){var i=new uj(t).startWalking();(0,lj.assign)(e,i)}),e}As.computeAllProdsFollows=SEe;function gj(r,e){return r.name+e+cj.IN}As.buildBetweenProdsFollowPrefix=gj;function vEe(r){var e=r.terminalType.name;return e+r.idx+cj.IN}As.buildInProdFollowPrefix=vEe});var Cd=w(Ba=>{"use strict";Object.defineProperty(Ba,"__esModule",{value:!0});Ba.defaultGrammarValidatorErrorProvider=Ba.defaultGrammarResolverErrorProvider=Ba.defaultParserErrorProvider=void 0;var jg=SA(),xEe=Gt(),Xs=Gt(),Rv=dn(),hj=dd();Ba.defaultParserErrorProvider={buildMismatchTokenMessage:function(r){var e=r.expected,t=r.actual,i=r.previous,n=r.ruleName,s=(0,jg.hasTokenLabel)(e),o=s?"--> "+(0,jg.tokenLabel)(e)+" <--":"token of type --> "+e.name+" <--",a="Expecting "+o+" but found --> '"+t.image+"' <--";return a},buildNotAllInputParsedMessage:function(r){var e=r.firstRedundant,t=r.ruleName;return"Redundant input, expecting EOF but found: "+e.image},buildNoViableAltMessage:function(r){var e=r.expectedPathsPerAlt,t=r.actual,i=r.previous,n=r.customUserDescription,s=r.ruleName,o="Expecting: ",a=(0,Xs.first)(t).image,l=` -but found: '`+a+"'";if(n)return o+n+l;var c=(0,Xs.reduce)(e,function(h,p){return h.concat(p)},[]),u=(0,Xs.map)(c,function(h){return"["+(0,Xs.map)(h,function(p){return(0,jg.tokenLabel)(p)}).join(", ")+"]"}),g=(0,Xs.map)(u,function(h,p){return" "+(p+1)+". "+h}),f=`one of these possible Token sequences: -`+g.join(` -`);return o+f+l},buildEarlyExitMessage:function(r){var e=r.expectedIterationPaths,t=r.actual,i=r.customUserDescription,n=r.ruleName,s="Expecting: ",o=(0,Xs.first)(t).image,a=` -but found: '`+o+"'";if(i)return s+i+a;var l=(0,Xs.map)(e,function(u){return"["+(0,Xs.map)(u,function(g){return(0,jg.tokenLabel)(g)}).join(",")+"]"}),c=`expecting at least one iteration which starts with one of these possible Token sequences:: - `+("<"+l.join(" ,")+">");return s+c+a}};Object.freeze(Ba.defaultParserErrorProvider);Ba.defaultGrammarResolverErrorProvider={buildRuleNotFoundError:function(r,e){var t="Invalid grammar, reference to a rule which is not defined: ->"+e.nonTerminalName+`<- -inside top level rule: ->`+r.name+"<-";return t}};Ba.defaultGrammarValidatorErrorProvider={buildDuplicateFoundError:function(r,e){function t(u){return u instanceof Rv.Terminal?u.terminalType.name:u instanceof Rv.NonTerminal?u.nonTerminalName:""}var i=r.name,n=(0,Xs.first)(e),s=n.idx,o=(0,hj.getProductionDslName)(n),a=t(n),l=s>0,c="->"+o+(l?s:"")+"<- "+(a?"with argument: ->"+a+"<-":"")+` - appears more than once (`+e.length+" times) in the top level rule: ->"+i+`<-. - For further details see: https://chevrotain.io/docs/FAQ.html#NUMERICAL_SUFFIXES - `;return c=c.replace(/[ \t]+/g," "),c=c.replace(/\s\s+/g,` -`),c},buildNamespaceConflictError:function(r){var e=`Namespace conflict found in grammar. -`+("The grammar has both a Terminal(Token) and a Non-Terminal(Rule) named: <"+r.name+`>. -`)+`To resolve this make sure each Terminal and Non-Terminal names are unique -This is easy to accomplish by using the convention that Terminal names start with an uppercase letter -and Non-Terminal names start with a lower case letter.`;return e},buildAlternationPrefixAmbiguityError:function(r){var e=(0,Xs.map)(r.prefixPath,function(n){return(0,jg.tokenLabel)(n)}).join(", "),t=r.alternation.idx===0?"":r.alternation.idx,i="Ambiguous alternatives: <"+r.ambiguityIndices.join(" ,")+`> due to common lookahead prefix -`+("in inside <"+r.topLevelRule.name+`> Rule, -`)+("<"+e+`> may appears as a prefix path in all these alternatives. -`)+`See: https://chevrotain.io/docs/guide/resolving_grammar_errors.html#COMMON_PREFIX -For Further details.`;return i},buildAlternationAmbiguityError:function(r){var e=(0,Xs.map)(r.prefixPath,function(n){return(0,jg.tokenLabel)(n)}).join(", "),t=r.alternation.idx===0?"":r.alternation.idx,i="Ambiguous Alternatives Detected: <"+r.ambiguityIndices.join(" ,")+"> in "+(" inside <"+r.topLevelRule.name+`> Rule, -`)+("<"+e+`> may appears as a prefix path in all these alternatives. -`);return i=i+`See: https://chevrotain.io/docs/guide/resolving_grammar_errors.html#AMBIGUOUS_ALTERNATIVES -For Further details.`,i},buildEmptyRepetitionError:function(r){var e=(0,hj.getProductionDslName)(r.repetition);r.repetition.idx!==0&&(e+=r.repetition.idx);var t="The repetition <"+e+"> within Rule <"+r.topLevelRule.name+`> can never consume any tokens. -This could lead to an infinite loop.`;return t},buildTokenNameError:function(r){return"deprecated"},buildEmptyAlternationError:function(r){var e="Ambiguous empty alternative: <"+(r.emptyChoiceIdx+1)+">"+(" in inside <"+r.topLevelRule.name+`> Rule. -`)+"Only the last alternative may be an empty alternative.";return e},buildTooManyAlternativesError:function(r){var e=`An Alternation cannot have more than 256 alternatives: -`+(" inside <"+r.topLevelRule.name+`> Rule. - has `+(r.alternation.definition.length+1)+" alternatives.");return e},buildLeftRecursionError:function(r){var e=r.topLevelRule.name,t=xEe.map(r.leftRecursionPath,function(s){return s.name}),i=e+" --> "+t.concat([e]).join(" --> "),n=`Left Recursion found in grammar. -`+("rule: <"+e+`> can be invoked from itself (directly or indirectly) -`)+(`without consuming any Tokens. The grammar path that causes this is: - `+i+` -`)+` To fix this refactor your grammar to remove the left recursion. -see: https://en.wikipedia.org/wiki/LL_parser#Left_Factoring.`;return n},buildInvalidRuleNameError:function(r){return"deprecated"},buildDuplicateRuleNameError:function(r){var e;r.topLevelRule instanceof Rv.Rule?e=r.topLevelRule.name:e=r.topLevelRule;var t="Duplicate definition, rule: ->"+e+"<- is already defined in the grammar: ->"+r.grammarName+"<-";return t}}});var Cj=w(vA=>{"use strict";var PEe=vA&&vA.__extends||function(){var r=function(e,t){return r=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(i,n){i.__proto__=n}||function(i,n){for(var s in n)Object.prototype.hasOwnProperty.call(n,s)&&(i[s]=n[s])},r(e,t)};return function(e,t){if(typeof t!="function"&&t!==null)throw new TypeError("Class extends value "+String(t)+" is not a constructor or null");r(e,t);function i(){this.constructor=e}e.prototype=t===null?Object.create(t):(i.prototype=t.prototype,new i)}}();Object.defineProperty(vA,"__esModule",{value:!0});vA.GastRefResolverVisitor=vA.resolveGrammar=void 0;var DEe=Hn(),pj=Gt(),kEe=Yg();function REe(r,e){var t=new dj(r,e);return t.resolveRefs(),t.errors}vA.resolveGrammar=REe;var dj=function(r){PEe(e,r);function e(t,i){var n=r.call(this)||this;return n.nameToTopRule=t,n.errMsgProvider=i,n.errors=[],n}return e.prototype.resolveRefs=function(){var t=this;(0,pj.forEach)((0,pj.values)(this.nameToTopRule),function(i){t.currTopLevel=i,i.accept(t)})},e.prototype.visitNonTerminal=function(t){var i=this.nameToTopRule[t.nonTerminalName];if(i)t.referencedRule=i;else{var n=this.errMsgProvider.buildRuleNotFoundError(this.currTopLevel,t);this.errors.push({message:n,type:DEe.ParserDefinitionErrorType.UNRESOLVED_SUBRULE_REF,ruleName:this.currTopLevel.name,unresolvedRefName:t.nonTerminalName})}},e}(kEe.GAstVisitor);vA.GastRefResolverVisitor=dj});var Ed=w(Nr=>{"use strict";var sc=Nr&&Nr.__extends||function(){var r=function(e,t){return r=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(i,n){i.__proto__=n}||function(i,n){for(var s in n)Object.prototype.hasOwnProperty.call(n,s)&&(i[s]=n[s])},r(e,t)};return function(e,t){if(typeof t!="function"&&t!==null)throw new TypeError("Class extends value "+String(t)+" is not a constructor or null");r(e,t);function i(){this.constructor=e}e.prototype=t===null?Object.create(t):(i.prototype=t.prototype,new i)}}();Object.defineProperty(Nr,"__esModule",{value:!0});Nr.nextPossibleTokensAfter=Nr.possiblePathsFrom=Nr.NextTerminalAfterAtLeastOneSepWalker=Nr.NextTerminalAfterAtLeastOneWalker=Nr.NextTerminalAfterManySepWalker=Nr.NextTerminalAfterManyWalker=Nr.AbstractNextTerminalAfterProductionWalker=Nr.NextAfterTokenWalker=Nr.AbstractNextPossibleTokensWalker=void 0;var mj=VI(),Kt=Gt(),FEe=Dv(),kt=dn(),Ej=function(r){sc(e,r);function e(t,i){var n=r.call(this)||this;return n.topProd=t,n.path=i,n.possibleTokTypes=[],n.nextProductionName="",n.nextProductionOccurrence=0,n.found=!1,n.isAtEndOfPath=!1,n}return e.prototype.startWalking=function(){if(this.found=!1,this.path.ruleStack[0]!==this.topProd.name)throw Error("The path does not start with the walker's top Rule!");return this.ruleStack=(0,Kt.cloneArr)(this.path.ruleStack).reverse(),this.occurrenceStack=(0,Kt.cloneArr)(this.path.occurrenceStack).reverse(),this.ruleStack.pop(),this.occurrenceStack.pop(),this.updateExpectedNext(),this.walk(this.topProd),this.possibleTokTypes},e.prototype.walk=function(t,i){i===void 0&&(i=[]),this.found||r.prototype.walk.call(this,t,i)},e.prototype.walkProdRef=function(t,i,n){if(t.referencedRule.name===this.nextProductionName&&t.idx===this.nextProductionOccurrence){var s=i.concat(n);this.updateExpectedNext(),this.walk(t.referencedRule,s)}},e.prototype.updateExpectedNext=function(){(0,Kt.isEmpty)(this.ruleStack)?(this.nextProductionName="",this.nextProductionOccurrence=0,this.isAtEndOfPath=!0):(this.nextProductionName=this.ruleStack.pop(),this.nextProductionOccurrence=this.occurrenceStack.pop())},e}(mj.RestWalker);Nr.AbstractNextPossibleTokensWalker=Ej;var NEe=function(r){sc(e,r);function e(t,i){var n=r.call(this,t,i)||this;return n.path=i,n.nextTerminalName="",n.nextTerminalOccurrence=0,n.nextTerminalName=n.path.lastTok.name,n.nextTerminalOccurrence=n.path.lastTokOccurrence,n}return e.prototype.walkTerminal=function(t,i,n){if(this.isAtEndOfPath&&t.terminalType.name===this.nextTerminalName&&t.idx===this.nextTerminalOccurrence&&!this.found){var s=i.concat(n),o=new kt.Alternative({definition:s});this.possibleTokTypes=(0,FEe.first)(o),this.found=!0}},e}(Ej);Nr.NextAfterTokenWalker=NEe;var md=function(r){sc(e,r);function e(t,i){var n=r.call(this)||this;return n.topRule=t,n.occurrence=i,n.result={token:void 0,occurrence:void 0,isEndOfRule:void 0},n}return e.prototype.startWalking=function(){return this.walk(this.topRule),this.result},e}(mj.RestWalker);Nr.AbstractNextTerminalAfterProductionWalker=md;var LEe=function(r){sc(e,r);function e(){return r!==null&&r.apply(this,arguments)||this}return e.prototype.walkMany=function(t,i,n){if(t.idx===this.occurrence){var s=(0,Kt.first)(i.concat(n));this.result.isEndOfRule=s===void 0,s instanceof kt.Terminal&&(this.result.token=s.terminalType,this.result.occurrence=s.idx)}else r.prototype.walkMany.call(this,t,i,n)},e}(md);Nr.NextTerminalAfterManyWalker=LEe;var TEe=function(r){sc(e,r);function e(){return r!==null&&r.apply(this,arguments)||this}return e.prototype.walkManySep=function(t,i,n){if(t.idx===this.occurrence){var s=(0,Kt.first)(i.concat(n));this.result.isEndOfRule=s===void 0,s instanceof kt.Terminal&&(this.result.token=s.terminalType,this.result.occurrence=s.idx)}else r.prototype.walkManySep.call(this,t,i,n)},e}(md);Nr.NextTerminalAfterManySepWalker=TEe;var OEe=function(r){sc(e,r);function e(){return r!==null&&r.apply(this,arguments)||this}return e.prototype.walkAtLeastOne=function(t,i,n){if(t.idx===this.occurrence){var s=(0,Kt.first)(i.concat(n));this.result.isEndOfRule=s===void 0,s instanceof kt.Terminal&&(this.result.token=s.terminalType,this.result.occurrence=s.idx)}else r.prototype.walkAtLeastOne.call(this,t,i,n)},e}(md);Nr.NextTerminalAfterAtLeastOneWalker=OEe;var MEe=function(r){sc(e,r);function e(){return r!==null&&r.apply(this,arguments)||this}return e.prototype.walkAtLeastOneSep=function(t,i,n){if(t.idx===this.occurrence){var s=(0,Kt.first)(i.concat(n));this.result.isEndOfRule=s===void 0,s instanceof kt.Terminal&&(this.result.token=s.terminalType,this.result.occurrence=s.idx)}else r.prototype.walkAtLeastOneSep.call(this,t,i,n)},e}(md);Nr.NextTerminalAfterAtLeastOneSepWalker=MEe;function Ij(r,e,t){t===void 0&&(t=[]),t=(0,Kt.cloneArr)(t);var i=[],n=0;function s(c){return c.concat((0,Kt.drop)(r,n+1))}function o(c){var u=Ij(s(c),e,t);return i.concat(u)}for(;t.length=0;ge--){var re=B.definition[ge],O={idx:p,def:re.definition.concat((0,Kt.drop)(h)),ruleStack:C,occurrenceStack:y};g.push(O),g.push(o)}else if(B instanceof kt.Alternative)g.push({idx:p,def:B.definition.concat((0,Kt.drop)(h)),ruleStack:C,occurrenceStack:y});else if(B instanceof kt.Rule)g.push(UEe(B,p,C,y));else throw Error("non exhaustive match")}}return u}Nr.nextPossibleTokensAfter=KEe;function UEe(r,e,t,i){var n=(0,Kt.cloneArr)(t);n.push(r.name);var s=(0,Kt.cloneArr)(i);return s.push(1),{idx:e,def:r.definition,ruleStack:n,occurrenceStack:s}}});var Id=w(_t=>{"use strict";var Bj=_t&&_t.__extends||function(){var r=function(e,t){return r=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(i,n){i.__proto__=n}||function(i,n){for(var s in n)Object.prototype.hasOwnProperty.call(n,s)&&(i[s]=n[s])},r(e,t)};return function(e,t){if(typeof t!="function"&&t!==null)throw new TypeError("Class extends value "+String(t)+" is not a constructor or null");r(e,t);function i(){this.constructor=e}e.prototype=t===null?Object.create(t):(i.prototype=t.prototype,new i)}}();Object.defineProperty(_t,"__esModule",{value:!0});_t.areTokenCategoriesNotUsed=_t.isStrictPrefixOfPath=_t.containsPath=_t.getLookaheadPathsForOptionalProd=_t.getLookaheadPathsForOr=_t.lookAheadSequenceFromAlternatives=_t.buildSingleAlternativeLookaheadFunction=_t.buildAlternativesLookAheadFunc=_t.buildLookaheadFuncForOptionalProd=_t.buildLookaheadFuncForOr=_t.getProdType=_t.PROD_TYPE=void 0;var sr=Gt(),yj=Ed(),HEe=VI(),ty=Gg(),xA=dn(),GEe=Yg(),oi;(function(r){r[r.OPTION=0]="OPTION",r[r.REPETITION=1]="REPETITION",r[r.REPETITION_MANDATORY=2]="REPETITION_MANDATORY",r[r.REPETITION_MANDATORY_WITH_SEPARATOR=3]="REPETITION_MANDATORY_WITH_SEPARATOR",r[r.REPETITION_WITH_SEPARATOR=4]="REPETITION_WITH_SEPARATOR",r[r.ALTERNATION=5]="ALTERNATION"})(oi=_t.PROD_TYPE||(_t.PROD_TYPE={}));function YEe(r){if(r instanceof xA.Option)return oi.OPTION;if(r instanceof xA.Repetition)return oi.REPETITION;if(r instanceof xA.RepetitionMandatory)return oi.REPETITION_MANDATORY;if(r instanceof xA.RepetitionMandatoryWithSeparator)return oi.REPETITION_MANDATORY_WITH_SEPARATOR;if(r instanceof xA.RepetitionWithSeparator)return oi.REPETITION_WITH_SEPARATOR;if(r instanceof xA.Alternation)return oi.ALTERNATION;throw Error("non exhaustive match")}_t.getProdType=YEe;function jEe(r,e,t,i,n,s){var o=bj(r,e,t),a=Lv(o)?ty.tokenStructuredMatcherNoCategories:ty.tokenStructuredMatcher;return s(o,i,a,n)}_t.buildLookaheadFuncForOr=jEe;function qEe(r,e,t,i,n,s){var o=Sj(r,e,n,t),a=Lv(o)?ty.tokenStructuredMatcherNoCategories:ty.tokenStructuredMatcher;return s(o[0],a,i)}_t.buildLookaheadFuncForOptionalProd=qEe;function JEe(r,e,t,i){var n=r.length,s=(0,sr.every)(r,function(l){return(0,sr.every)(l,function(c){return c.length===1})});if(e)return function(l){for(var c=(0,sr.map)(l,function(D){return D.GATE}),u=0;u{"use strict";var Tv=zt&&zt.__extends||function(){var r=function(e,t){return r=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(i,n){i.__proto__=n}||function(i,n){for(var s in n)Object.prototype.hasOwnProperty.call(n,s)&&(i[s]=n[s])},r(e,t)};return function(e,t){if(typeof t!="function"&&t!==null)throw new TypeError("Class extends value "+String(t)+" is not a constructor or null");r(e,t);function i(){this.constructor=e}e.prototype=t===null?Object.create(t):(i.prototype=t.prototype,new i)}}();Object.defineProperty(zt,"__esModule",{value:!0});zt.checkPrefixAlternativesAmbiguities=zt.validateSomeNonEmptyLookaheadPath=zt.validateTooManyAlts=zt.RepetionCollector=zt.validateAmbiguousAlternationAlternatives=zt.validateEmptyOrAlternative=zt.getFirstNoneTerminal=zt.validateNoLeftRecursion=zt.validateRuleIsOverridden=zt.validateRuleDoesNotAlreadyExist=zt.OccurrenceValidationCollector=zt.identifyProductionForDuplicates=zt.validateGrammar=void 0;var er=Gt(),br=Gt(),Do=Hn(),Ov=dd(),qg=Id(),_Ee=Ed(),_s=dn(),Mv=Yg();function ZEe(r,e,t,i,n){var s=er.map(r,function(h){return $Ee(h,i)}),o=er.map(r,function(h){return Kv(h,h,i)}),a=[],l=[],c=[];(0,br.every)(o,br.isEmpty)&&(a=(0,br.map)(r,function(h){return Rj(h,i)}),l=(0,br.map)(r,function(h){return Fj(h,e,i)}),c=Tj(r,e,i));var u=rIe(r,t,i),g=(0,br.map)(r,function(h){return Lj(h,i)}),f=(0,br.map)(r,function(h){return kj(h,r,n,i)});return er.flatten(s.concat(c,o,a,l,u,g,f))}zt.validateGrammar=ZEe;function $Ee(r,e){var t=new Dj;r.accept(t);var i=t.allProductions,n=er.groupBy(i,xj),s=er.pick(n,function(a){return a.length>1}),o=er.map(er.values(s),function(a){var l=er.first(a),c=e.buildDuplicateFoundError(r,a),u=(0,Ov.getProductionDslName)(l),g={message:c,type:Do.ParserDefinitionErrorType.DUPLICATE_PRODUCTIONS,ruleName:r.name,dslName:u,occurrence:l.idx},f=Pj(l);return f&&(g.parameter=f),g});return o}function xj(r){return(0,Ov.getProductionDslName)(r)+"_#_"+r.idx+"_#_"+Pj(r)}zt.identifyProductionForDuplicates=xj;function Pj(r){return r instanceof _s.Terminal?r.terminalType.name:r instanceof _s.NonTerminal?r.nonTerminalName:""}var Dj=function(r){Tv(e,r);function e(){var t=r!==null&&r.apply(this,arguments)||this;return t.allProductions=[],t}return e.prototype.visitNonTerminal=function(t){this.allProductions.push(t)},e.prototype.visitOption=function(t){this.allProductions.push(t)},e.prototype.visitRepetitionWithSeparator=function(t){this.allProductions.push(t)},e.prototype.visitRepetitionMandatory=function(t){this.allProductions.push(t)},e.prototype.visitRepetitionMandatoryWithSeparator=function(t){this.allProductions.push(t)},e.prototype.visitRepetition=function(t){this.allProductions.push(t)},e.prototype.visitAlternation=function(t){this.allProductions.push(t)},e.prototype.visitTerminal=function(t){this.allProductions.push(t)},e}(Mv.GAstVisitor);zt.OccurrenceValidationCollector=Dj;function kj(r,e,t,i){var n=[],s=(0,br.reduce)(e,function(a,l){return l.name===r.name?a+1:a},0);if(s>1){var o=i.buildDuplicateRuleNameError({topLevelRule:r,grammarName:t});n.push({message:o,type:Do.ParserDefinitionErrorType.DUPLICATE_RULE_NAME,ruleName:r.name})}return n}zt.validateRuleDoesNotAlreadyExist=kj;function eIe(r,e,t){var i=[],n;return er.contains(e,r)||(n="Invalid rule override, rule: ->"+r+"<- cannot be overridden in the grammar: ->"+t+"<-as it is not defined in any of the super grammars ",i.push({message:n,type:Do.ParserDefinitionErrorType.INVALID_RULE_OVERRIDE,ruleName:r})),i}zt.validateRuleIsOverridden=eIe;function Kv(r,e,t,i){i===void 0&&(i=[]);var n=[],s=yd(e.definition);if(er.isEmpty(s))return[];var o=r.name,a=er.contains(s,r);a&&n.push({message:t.buildLeftRecursionError({topLevelRule:r,leftRecursionPath:i}),type:Do.ParserDefinitionErrorType.LEFT_RECURSION,ruleName:o});var l=er.difference(s,i.concat([r])),c=er.map(l,function(u){var g=er.cloneArr(i);return g.push(u),Kv(r,u,t,g)});return n.concat(er.flatten(c))}zt.validateNoLeftRecursion=Kv;function yd(r){var e=[];if(er.isEmpty(r))return e;var t=er.first(r);if(t instanceof _s.NonTerminal)e.push(t.referencedRule);else if(t instanceof _s.Alternative||t instanceof _s.Option||t instanceof _s.RepetitionMandatory||t instanceof _s.RepetitionMandatoryWithSeparator||t instanceof _s.RepetitionWithSeparator||t instanceof _s.Repetition)e=e.concat(yd(t.definition));else if(t instanceof _s.Alternation)e=er.flatten(er.map(t.definition,function(o){return yd(o.definition)}));else if(!(t instanceof _s.Terminal))throw Error("non exhaustive match");var i=(0,Ov.isOptionalProd)(t),n=r.length>1;if(i&&n){var s=er.drop(r);return e.concat(yd(s))}else return e}zt.getFirstNoneTerminal=yd;var Uv=function(r){Tv(e,r);function e(){var t=r!==null&&r.apply(this,arguments)||this;return t.alternations=[],t}return e.prototype.visitAlternation=function(t){this.alternations.push(t)},e}(Mv.GAstVisitor);function Rj(r,e){var t=new Uv;r.accept(t);var i=t.alternations,n=er.reduce(i,function(s,o){var a=er.dropRight(o.definition),l=er.map(a,function(c,u){var g=(0,_Ee.nextPossibleTokensAfter)([c],[],null,1);return er.isEmpty(g)?{message:e.buildEmptyAlternationError({topLevelRule:r,alternation:o,emptyChoiceIdx:u}),type:Do.ParserDefinitionErrorType.NONE_LAST_EMPTY_ALT,ruleName:r.name,occurrence:o.idx,alternative:u+1}:null});return s.concat(er.compact(l))},[]);return n}zt.validateEmptyOrAlternative=Rj;function Fj(r,e,t){var i=new Uv;r.accept(i);var n=i.alternations;n=(0,br.reject)(n,function(o){return o.ignoreAmbiguities===!0});var s=er.reduce(n,function(o,a){var l=a.idx,c=a.maxLookahead||e,u=(0,qg.getLookaheadPathsForOr)(l,r,c,a),g=tIe(u,a,r,t),f=Oj(u,a,r,t);return o.concat(g,f)},[]);return s}zt.validateAmbiguousAlternationAlternatives=Fj;var Nj=function(r){Tv(e,r);function e(){var t=r!==null&&r.apply(this,arguments)||this;return t.allProductions=[],t}return e.prototype.visitRepetitionWithSeparator=function(t){this.allProductions.push(t)},e.prototype.visitRepetitionMandatory=function(t){this.allProductions.push(t)},e.prototype.visitRepetitionMandatoryWithSeparator=function(t){this.allProductions.push(t)},e.prototype.visitRepetition=function(t){this.allProductions.push(t)},e}(Mv.GAstVisitor);zt.RepetionCollector=Nj;function Lj(r,e){var t=new Uv;r.accept(t);var i=t.alternations,n=er.reduce(i,function(s,o){return o.definition.length>255&&s.push({message:e.buildTooManyAlternativesError({topLevelRule:r,alternation:o}),type:Do.ParserDefinitionErrorType.TOO_MANY_ALTS,ruleName:r.name,occurrence:o.idx}),s},[]);return n}zt.validateTooManyAlts=Lj;function Tj(r,e,t){var i=[];return(0,br.forEach)(r,function(n){var s=new Nj;n.accept(s);var o=s.allProductions;(0,br.forEach)(o,function(a){var l=(0,qg.getProdType)(a),c=a.maxLookahead||e,u=a.idx,g=(0,qg.getLookaheadPathsForOptionalProd)(u,n,l,c),f=g[0];if((0,br.isEmpty)((0,br.flatten)(f))){var h=t.buildEmptyRepetitionError({topLevelRule:n,repetition:a});i.push({message:h,type:Do.ParserDefinitionErrorType.NO_NON_EMPTY_LOOKAHEAD,ruleName:n.name})}})}),i}zt.validateSomeNonEmptyLookaheadPath=Tj;function tIe(r,e,t,i){var n=[],s=(0,br.reduce)(r,function(a,l,c){return e.definition[c].ignoreAmbiguities===!0||(0,br.forEach)(l,function(u){var g=[c];(0,br.forEach)(r,function(f,h){c!==h&&(0,qg.containsPath)(f,u)&&e.definition[h].ignoreAmbiguities!==!0&&g.push(h)}),g.length>1&&!(0,qg.containsPath)(n,u)&&(n.push(u),a.push({alts:g,path:u}))}),a},[]),o=er.map(s,function(a){var l=(0,br.map)(a.alts,function(u){return u+1}),c=i.buildAlternationAmbiguityError({topLevelRule:t,alternation:e,ambiguityIndices:l,prefixPath:a.path});return{message:c,type:Do.ParserDefinitionErrorType.AMBIGUOUS_ALTS,ruleName:t.name,occurrence:e.idx,alternatives:[a.alts]}});return o}function Oj(r,e,t,i){var n=[],s=(0,br.reduce)(r,function(o,a,l){var c=(0,br.map)(a,function(u){return{idx:l,path:u}});return o.concat(c)},[]);return(0,br.forEach)(s,function(o){var a=e.definition[o.idx];if(a.ignoreAmbiguities!==!0){var l=o.idx,c=o.path,u=(0,br.findAll)(s,function(f){return e.definition[f.idx].ignoreAmbiguities!==!0&&f.idx{"use strict";Object.defineProperty(Jg,"__esModule",{value:!0});Jg.validateGrammar=Jg.resolveGrammar=void 0;var Gv=Gt(),iIe=Cj(),nIe=Hv(),Mj=Cd();function sIe(r){r=(0,Gv.defaults)(r,{errMsgProvider:Mj.defaultGrammarResolverErrorProvider});var e={};return(0,Gv.forEach)(r.rules,function(t){e[t.name]=t}),(0,iIe.resolveGrammar)(e,r.errMsgProvider)}Jg.resolveGrammar=sIe;function oIe(r){return r=(0,Gv.defaults)(r,{errMsgProvider:Mj.defaultGrammarValidatorErrorProvider}),(0,nIe.validateGrammar)(r.rules,r.maxLookahead,r.tokenTypes,r.errMsgProvider,r.grammarName)}Jg.validateGrammar=oIe});var Wg=w(mn=>{"use strict";var wd=mn&&mn.__extends||function(){var r=function(e,t){return r=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(i,n){i.__proto__=n}||function(i,n){for(var s in n)Object.prototype.hasOwnProperty.call(n,s)&&(i[s]=n[s])},r(e,t)};return function(e,t){if(typeof t!="function"&&t!==null)throw new TypeError("Class extends value "+String(t)+" is not a constructor or null");r(e,t);function i(){this.constructor=e}e.prototype=t===null?Object.create(t):(i.prototype=t.prototype,new i)}}();Object.defineProperty(mn,"__esModule",{value:!0});mn.EarlyExitException=mn.NotAllInputParsedException=mn.NoViableAltException=mn.MismatchedTokenException=mn.isRecognitionException=void 0;var aIe=Gt(),Uj="MismatchedTokenException",Hj="NoViableAltException",Gj="EarlyExitException",Yj="NotAllInputParsedException",jj=[Uj,Hj,Gj,Yj];Object.freeze(jj);function AIe(r){return(0,aIe.contains)(jj,r.name)}mn.isRecognitionException=AIe;var ry=function(r){wd(e,r);function e(t,i){var n=this.constructor,s=r.call(this,t)||this;return s.token=i,s.resyncedTokens=[],Object.setPrototypeOf(s,n.prototype),Error.captureStackTrace&&Error.captureStackTrace(s,s.constructor),s}return e}(Error),lIe=function(r){wd(e,r);function e(t,i,n){var s=r.call(this,t,i)||this;return s.previousToken=n,s.name=Uj,s}return e}(ry);mn.MismatchedTokenException=lIe;var cIe=function(r){wd(e,r);function e(t,i,n){var s=r.call(this,t,i)||this;return s.previousToken=n,s.name=Hj,s}return e}(ry);mn.NoViableAltException=cIe;var uIe=function(r){wd(e,r);function e(t,i){var n=r.call(this,t,i)||this;return n.name=Yj,n}return e}(ry);mn.NotAllInputParsedException=uIe;var gIe=function(r){wd(e,r);function e(t,i,n){var s=r.call(this,t,i)||this;return s.previousToken=n,s.name=Gj,s}return e}(ry);mn.EarlyExitException=gIe});var jv=w(Mi=>{"use strict";Object.defineProperty(Mi,"__esModule",{value:!0});Mi.attemptInRepetitionRecovery=Mi.Recoverable=Mi.InRuleRecoveryException=Mi.IN_RULE_RECOVERY_EXCEPTION=Mi.EOF_FOLLOW_KEY=void 0;var iy=SA(),ls=Gt(),fIe=Wg(),hIe=kv(),pIe=Hn();Mi.EOF_FOLLOW_KEY={};Mi.IN_RULE_RECOVERY_EXCEPTION="InRuleRecoveryException";function Yv(r){this.name=Mi.IN_RULE_RECOVERY_EXCEPTION,this.message=r}Mi.InRuleRecoveryException=Yv;Yv.prototype=Error.prototype;var dIe=function(){function r(){}return r.prototype.initRecoverable=function(e){this.firstAfterRepMap={},this.resyncFollows={},this.recoveryEnabled=(0,ls.has)(e,"recoveryEnabled")?e.recoveryEnabled:pIe.DEFAULT_PARSER_CONFIG.recoveryEnabled,this.recoveryEnabled&&(this.attemptInRepetitionRecovery=qj)},r.prototype.getTokenToInsert=function(e){var t=(0,iy.createTokenInstance)(e,"",NaN,NaN,NaN,NaN,NaN,NaN);return t.isInsertedInRecovery=!0,t},r.prototype.canTokenTypeBeInsertedInRecovery=function(e){return!0},r.prototype.tryInRepetitionRecovery=function(e,t,i,n){for(var s=this,o=this.findReSyncTokenType(),a=this.exportLexerState(),l=[],c=!1,u=this.LA(1),g=this.LA(1),f=function(){var h=s.LA(0),p=s.errorMessageProvider.buildMismatchTokenMessage({expected:n,actual:u,previous:h,ruleName:s.getCurrRuleFullName()}),C=new fIe.MismatchedTokenException(p,u,s.LA(0));C.resyncedTokens=(0,ls.dropRight)(l),s.SAVE_ERROR(C)};!c;)if(this.tokenMatcher(g,n)){f();return}else if(i.call(this)){f(),e.apply(this,t);return}else this.tokenMatcher(g,o)?c=!0:(g=this.SKIP_TOKEN(),this.addToResyncTokens(g,l));this.importLexerState(a)},r.prototype.shouldInRepetitionRecoveryBeTried=function(e,t,i){return!(i===!1||e===void 0||t===void 0||this.tokenMatcher(this.LA(1),e)||this.isBackTracking()||this.canPerformInRuleRecovery(e,this.getFollowsForInRuleRecovery(e,t)))},r.prototype.getFollowsForInRuleRecovery=function(e,t){var i=this.getCurrentGrammarPath(e,t),n=this.getNextPossibleTokenTypes(i);return n},r.prototype.tryInRuleRecovery=function(e,t){if(this.canRecoverWithSingleTokenInsertion(e,t)){var i=this.getTokenToInsert(e);return i}if(this.canRecoverWithSingleTokenDeletion(e)){var n=this.SKIP_TOKEN();return this.consumeToken(),n}throw new Yv("sad sad panda")},r.prototype.canPerformInRuleRecovery=function(e,t){return this.canRecoverWithSingleTokenInsertion(e,t)||this.canRecoverWithSingleTokenDeletion(e)},r.prototype.canRecoverWithSingleTokenInsertion=function(e,t){var i=this;if(!this.canTokenTypeBeInsertedInRecovery(e)||(0,ls.isEmpty)(t))return!1;var n=this.LA(1),s=(0,ls.find)(t,function(o){return i.tokenMatcher(n,o)})!==void 0;return s},r.prototype.canRecoverWithSingleTokenDeletion=function(e){var t=this.tokenMatcher(this.LA(2),e);return t},r.prototype.isInCurrentRuleReSyncSet=function(e){var t=this.getCurrFollowKey(),i=this.getFollowSetFromFollowKey(t);return(0,ls.contains)(i,e)},r.prototype.findReSyncTokenType=function(){for(var e=this.flattenFollowSet(),t=this.LA(1),i=2;;){var n=t.tokenType;if((0,ls.contains)(e,n))return n;t=this.LA(i),i++}},r.prototype.getCurrFollowKey=function(){if(this.RULE_STACK.length===1)return Mi.EOF_FOLLOW_KEY;var e=this.getLastExplicitRuleShortName(),t=this.getLastExplicitRuleOccurrenceIndex(),i=this.getPreviousExplicitRuleShortName();return{ruleName:this.shortRuleNameToFullName(e),idxInCallingRule:t,inRule:this.shortRuleNameToFullName(i)}},r.prototype.buildFullFollowKeyStack=function(){var e=this,t=this.RULE_STACK,i=this.RULE_OCCURRENCE_STACK;return(0,ls.map)(t,function(n,s){return s===0?Mi.EOF_FOLLOW_KEY:{ruleName:e.shortRuleNameToFullName(n),idxInCallingRule:i[s],inRule:e.shortRuleNameToFullName(t[s-1])}})},r.prototype.flattenFollowSet=function(){var e=this,t=(0,ls.map)(this.buildFullFollowKeyStack(),function(i){return e.getFollowSetFromFollowKey(i)});return(0,ls.flatten)(t)},r.prototype.getFollowSetFromFollowKey=function(e){if(e===Mi.EOF_FOLLOW_KEY)return[iy.EOF];var t=e.ruleName+e.idxInCallingRule+hIe.IN+e.inRule;return this.resyncFollows[t]},r.prototype.addToResyncTokens=function(e,t){return this.tokenMatcher(e,iy.EOF)||t.push(e),t},r.prototype.reSyncTo=function(e){for(var t=[],i=this.LA(1);this.tokenMatcher(i,e)===!1;)i=this.SKIP_TOKEN(),this.addToResyncTokens(i,t);return(0,ls.dropRight)(t)},r.prototype.attemptInRepetitionRecovery=function(e,t,i,n,s,o,a){},r.prototype.getCurrentGrammarPath=function(e,t){var i=this.getHumanReadableRuleStack(),n=(0,ls.cloneArr)(this.RULE_OCCURRENCE_STACK),s={ruleStack:i,occurrenceStack:n,lastTok:e,lastTokOccurrence:t};return s},r.prototype.getHumanReadableRuleStack=function(){var e=this;return(0,ls.map)(this.RULE_STACK,function(t){return e.shortRuleNameToFullName(t)})},r}();Mi.Recoverable=dIe;function qj(r,e,t,i,n,s,o){var a=this.getKeyForAutomaticLookahead(i,n),l=this.firstAfterRepMap[a];if(l===void 0){var c=this.getCurrRuleFullName(),u=this.getGAstProductions()[c],g=new s(u,n);l=g.startWalking(),this.firstAfterRepMap[a]=l}var f=l.token,h=l.occurrence,p=l.isEndOfRule;this.RULE_STACK.length===1&&p&&f===void 0&&(f=iy.EOF,h=1),this.shouldInRepetitionRecoveryBeTried(f,h,o)&&this.tryInRepetitionRecovery(r,e,t,f)}Mi.attemptInRepetitionRecovery=qj});var ny=w(qt=>{"use strict";Object.defineProperty(qt,"__esModule",{value:!0});qt.getKeyForAutomaticLookahead=qt.AT_LEAST_ONE_SEP_IDX=qt.MANY_SEP_IDX=qt.AT_LEAST_ONE_IDX=qt.MANY_IDX=qt.OPTION_IDX=qt.OR_IDX=qt.BITS_FOR_ALT_IDX=qt.BITS_FOR_RULE_IDX=qt.BITS_FOR_OCCURRENCE_IDX=qt.BITS_FOR_METHOD_TYPE=void 0;qt.BITS_FOR_METHOD_TYPE=4;qt.BITS_FOR_OCCURRENCE_IDX=8;qt.BITS_FOR_RULE_IDX=12;qt.BITS_FOR_ALT_IDX=8;qt.OR_IDX=1<{"use strict";Object.defineProperty(sy,"__esModule",{value:!0});sy.LooksAhead=void 0;var Qa=Id(),Zs=Gt(),Jj=Hn(),ba=ny(),oc=dd(),mIe=function(){function r(){}return r.prototype.initLooksAhead=function(e){this.dynamicTokensEnabled=(0,Zs.has)(e,"dynamicTokensEnabled")?e.dynamicTokensEnabled:Jj.DEFAULT_PARSER_CONFIG.dynamicTokensEnabled,this.maxLookahead=(0,Zs.has)(e,"maxLookahead")?e.maxLookahead:Jj.DEFAULT_PARSER_CONFIG.maxLookahead,this.lookAheadFuncsCache=(0,Zs.isES2015MapSupported)()?new Map:[],(0,Zs.isES2015MapSupported)()?(this.getLaFuncFromCache=this.getLaFuncFromMap,this.setLaFuncCache=this.setLaFuncCacheUsingMap):(this.getLaFuncFromCache=this.getLaFuncFromObj,this.setLaFuncCache=this.setLaFuncUsingObj)},r.prototype.preComputeLookaheadFunctions=function(e){var t=this;(0,Zs.forEach)(e,function(i){t.TRACE_INIT(i.name+" Rule Lookahead",function(){var n=(0,oc.collectMethods)(i),s=n.alternation,o=n.repetition,a=n.option,l=n.repetitionMandatory,c=n.repetitionMandatoryWithSeparator,u=n.repetitionWithSeparator;(0,Zs.forEach)(s,function(g){var f=g.idx===0?"":g.idx;t.TRACE_INIT(""+(0,oc.getProductionDslName)(g)+f,function(){var h=(0,Qa.buildLookaheadFuncForOr)(g.idx,i,g.maxLookahead||t.maxLookahead,g.hasPredicates,t.dynamicTokensEnabled,t.lookAheadBuilderForAlternatives),p=(0,ba.getKeyForAutomaticLookahead)(t.fullRuleNameToShort[i.name],ba.OR_IDX,g.idx);t.setLaFuncCache(p,h)})}),(0,Zs.forEach)(o,function(g){t.computeLookaheadFunc(i,g.idx,ba.MANY_IDX,Qa.PROD_TYPE.REPETITION,g.maxLookahead,(0,oc.getProductionDslName)(g))}),(0,Zs.forEach)(a,function(g){t.computeLookaheadFunc(i,g.idx,ba.OPTION_IDX,Qa.PROD_TYPE.OPTION,g.maxLookahead,(0,oc.getProductionDslName)(g))}),(0,Zs.forEach)(l,function(g){t.computeLookaheadFunc(i,g.idx,ba.AT_LEAST_ONE_IDX,Qa.PROD_TYPE.REPETITION_MANDATORY,g.maxLookahead,(0,oc.getProductionDslName)(g))}),(0,Zs.forEach)(c,function(g){t.computeLookaheadFunc(i,g.idx,ba.AT_LEAST_ONE_SEP_IDX,Qa.PROD_TYPE.REPETITION_MANDATORY_WITH_SEPARATOR,g.maxLookahead,(0,oc.getProductionDslName)(g))}),(0,Zs.forEach)(u,function(g){t.computeLookaheadFunc(i,g.idx,ba.MANY_SEP_IDX,Qa.PROD_TYPE.REPETITION_WITH_SEPARATOR,g.maxLookahead,(0,oc.getProductionDslName)(g))})})})},r.prototype.computeLookaheadFunc=function(e,t,i,n,s,o){var a=this;this.TRACE_INIT(""+o+(t===0?"":t),function(){var l=(0,Qa.buildLookaheadFuncForOptionalProd)(t,e,s||a.maxLookahead,a.dynamicTokensEnabled,n,a.lookAheadBuilderForOptional),c=(0,ba.getKeyForAutomaticLookahead)(a.fullRuleNameToShort[e.name],i,t);a.setLaFuncCache(c,l)})},r.prototype.lookAheadBuilderForOptional=function(e,t,i){return(0,Qa.buildSingleAlternativeLookaheadFunction)(e,t,i)},r.prototype.lookAheadBuilderForAlternatives=function(e,t,i,n){return(0,Qa.buildAlternativesLookAheadFunc)(e,t,i,n)},r.prototype.getKeyForAutomaticLookahead=function(e,t){var i=this.getLastExplicitRuleShortName();return(0,ba.getKeyForAutomaticLookahead)(i,e,t)},r.prototype.getLaFuncFromCache=function(e){},r.prototype.getLaFuncFromMap=function(e){return this.lookAheadFuncsCache.get(e)},r.prototype.getLaFuncFromObj=function(e){return this.lookAheadFuncsCache[e]},r.prototype.setLaFuncCache=function(e,t){},r.prototype.setLaFuncCacheUsingMap=function(e,t){this.lookAheadFuncsCache.set(e,t)},r.prototype.setLaFuncUsingObj=function(e,t){this.lookAheadFuncsCache[e]=t},r}();sy.LooksAhead=mIe});var zj=w(ko=>{"use strict";Object.defineProperty(ko,"__esModule",{value:!0});ko.addNoneTerminalToCst=ko.addTerminalToCst=ko.setNodeLocationFull=ko.setNodeLocationOnlyOffset=void 0;function EIe(r,e){isNaN(r.startOffset)===!0?(r.startOffset=e.startOffset,r.endOffset=e.endOffset):r.endOffset{"use strict";Object.defineProperty(PA,"__esModule",{value:!0});PA.defineNameProp=PA.functionName=PA.classNameFromInstance=void 0;var BIe=Gt();function QIe(r){return Xj(r.constructor)}PA.classNameFromInstance=QIe;var Vj="name";function Xj(r){var e=r.name;return e||"anonymous"}PA.functionName=Xj;function bIe(r,e){var t=Object.getOwnPropertyDescriptor(r,Vj);return(0,BIe.isUndefined)(t)||t.configurable?(Object.defineProperty(r,Vj,{enumerable:!1,configurable:!0,writable:!1,value:e}),!0):!1}PA.defineNameProp=bIe});var tq=w(bi=>{"use strict";Object.defineProperty(bi,"__esModule",{value:!0});bi.validateRedundantMethods=bi.validateMissingCstMethods=bi.validateVisitor=bi.CstVisitorDefinitionError=bi.createBaseVisitorConstructorWithDefaults=bi.createBaseSemanticVisitorConstructor=bi.defaultVisit=void 0;var cs=Gt(),Bd=qv();function _j(r,e){for(var t=(0,cs.keys)(r),i=t.length,n=0;n: - `+(""+s.join(` - -`).replace(/\n/g,` - `)))}}};return t.prototype=i,t.prototype.constructor=t,t._RULE_NAMES=e,t}bi.createBaseSemanticVisitorConstructor=SIe;function vIe(r,e,t){var i=function(){};(0,Bd.defineNameProp)(i,r+"BaseSemanticsWithDefaults");var n=Object.create(t.prototype);return(0,cs.forEach)(e,function(s){n[s]=_j}),i.prototype=n,i.prototype.constructor=i,i}bi.createBaseVisitorConstructorWithDefaults=vIe;var Jv;(function(r){r[r.REDUNDANT_METHOD=0]="REDUNDANT_METHOD",r[r.MISSING_METHOD=1]="MISSING_METHOD"})(Jv=bi.CstVisitorDefinitionError||(bi.CstVisitorDefinitionError={}));function Zj(r,e){var t=$j(r,e),i=eq(r,e);return t.concat(i)}bi.validateVisitor=Zj;function $j(r,e){var t=(0,cs.map)(e,function(i){if(!(0,cs.isFunction)(r[i]))return{msg:"Missing visitor method: <"+i+"> on "+(0,Bd.functionName)(r.constructor)+" CST Visitor.",type:Jv.MISSING_METHOD,methodName:i}});return(0,cs.compact)(t)}bi.validateMissingCstMethods=$j;var xIe=["constructor","visit","validateVisitor"];function eq(r,e){var t=[];for(var i in r)(0,cs.isFunction)(r[i])&&!(0,cs.contains)(xIe,i)&&!(0,cs.contains)(e,i)&&t.push({msg:"Redundant visitor method: <"+i+"> on "+(0,Bd.functionName)(r.constructor)+` CST Visitor -There is no Grammar Rule corresponding to this method's name. -`,type:Jv.REDUNDANT_METHOD,methodName:i});return t}bi.validateRedundantMethods=eq});var iq=w(oy=>{"use strict";Object.defineProperty(oy,"__esModule",{value:!0});oy.TreeBuilder=void 0;var zg=zj(),Zr=Gt(),rq=tq(),PIe=Hn(),DIe=function(){function r(){}return r.prototype.initTreeBuilder=function(e){if(this.CST_STACK=[],this.outputCst=e.outputCst,this.nodeLocationTracking=(0,Zr.has)(e,"nodeLocationTracking")?e.nodeLocationTracking:PIe.DEFAULT_PARSER_CONFIG.nodeLocationTracking,!this.outputCst)this.cstInvocationStateUpdate=Zr.NOOP,this.cstFinallyStateUpdate=Zr.NOOP,this.cstPostTerminal=Zr.NOOP,this.cstPostNonTerminal=Zr.NOOP,this.cstPostRule=Zr.NOOP;else if(/full/i.test(this.nodeLocationTracking))this.recoveryEnabled?(this.setNodeLocationFromToken=zg.setNodeLocationFull,this.setNodeLocationFromNode=zg.setNodeLocationFull,this.cstPostRule=Zr.NOOP,this.setInitialNodeLocation=this.setInitialNodeLocationFullRecovery):(this.setNodeLocationFromToken=Zr.NOOP,this.setNodeLocationFromNode=Zr.NOOP,this.cstPostRule=this.cstPostRuleFull,this.setInitialNodeLocation=this.setInitialNodeLocationFullRegular);else if(/onlyOffset/i.test(this.nodeLocationTracking))this.recoveryEnabled?(this.setNodeLocationFromToken=zg.setNodeLocationOnlyOffset,this.setNodeLocationFromNode=zg.setNodeLocationOnlyOffset,this.cstPostRule=Zr.NOOP,this.setInitialNodeLocation=this.setInitialNodeLocationOnlyOffsetRecovery):(this.setNodeLocationFromToken=Zr.NOOP,this.setNodeLocationFromNode=Zr.NOOP,this.cstPostRule=this.cstPostRuleOnlyOffset,this.setInitialNodeLocation=this.setInitialNodeLocationOnlyOffsetRegular);else if(/none/i.test(this.nodeLocationTracking))this.setNodeLocationFromToken=Zr.NOOP,this.setNodeLocationFromNode=Zr.NOOP,this.cstPostRule=Zr.NOOP,this.setInitialNodeLocation=Zr.NOOP;else throw Error('Invalid config option: "'+e.nodeLocationTracking+'"')},r.prototype.setInitialNodeLocationOnlyOffsetRecovery=function(e){e.location={startOffset:NaN,endOffset:NaN}},r.prototype.setInitialNodeLocationOnlyOffsetRegular=function(e){e.location={startOffset:this.LA(1).startOffset,endOffset:NaN}},r.prototype.setInitialNodeLocationFullRecovery=function(e){e.location={startOffset:NaN,startLine:NaN,startColumn:NaN,endOffset:NaN,endLine:NaN,endColumn:NaN}},r.prototype.setInitialNodeLocationFullRegular=function(e){var t=this.LA(1);e.location={startOffset:t.startOffset,startLine:t.startLine,startColumn:t.startColumn,endOffset:NaN,endLine:NaN,endColumn:NaN}},r.prototype.cstInvocationStateUpdate=function(e,t){var i={name:e,children:{}};this.setInitialNodeLocation(i),this.CST_STACK.push(i)},r.prototype.cstFinallyStateUpdate=function(){this.CST_STACK.pop()},r.prototype.cstPostRuleFull=function(e){var t=this.LA(0),i=e.location;i.startOffset<=t.startOffset?(i.endOffset=t.endOffset,i.endLine=t.endLine,i.endColumn=t.endColumn):(i.startOffset=NaN,i.startLine=NaN,i.startColumn=NaN)},r.prototype.cstPostRuleOnlyOffset=function(e){var t=this.LA(0),i=e.location;i.startOffset<=t.startOffset?i.endOffset=t.endOffset:i.startOffset=NaN},r.prototype.cstPostTerminal=function(e,t){var i=this.CST_STACK[this.CST_STACK.length-1];(0,zg.addTerminalToCst)(i,t,e),this.setNodeLocationFromToken(i.location,t)},r.prototype.cstPostNonTerminal=function(e,t){var i=this.CST_STACK[this.CST_STACK.length-1];(0,zg.addNoneTerminalToCst)(i,t,e),this.setNodeLocationFromNode(i.location,e.location)},r.prototype.getBaseCstVisitorConstructor=function(){if((0,Zr.isUndefined)(this.baseCstVisitorConstructor)){var e=(0,rq.createBaseSemanticVisitorConstructor)(this.className,(0,Zr.keys)(this.gastProductionsCache));return this.baseCstVisitorConstructor=e,e}return this.baseCstVisitorConstructor},r.prototype.getBaseCstVisitorConstructorWithDefaults=function(){if((0,Zr.isUndefined)(this.baseCstVisitorWithDefaultsConstructor)){var e=(0,rq.createBaseVisitorConstructorWithDefaults)(this.className,(0,Zr.keys)(this.gastProductionsCache),this.getBaseCstVisitorConstructor());return this.baseCstVisitorWithDefaultsConstructor=e,e}return this.baseCstVisitorWithDefaultsConstructor},r.prototype.getLastExplicitRuleShortName=function(){var e=this.RULE_STACK;return e[e.length-1]},r.prototype.getPreviousExplicitRuleShortName=function(){var e=this.RULE_STACK;return e[e.length-2]},r.prototype.getLastExplicitRuleOccurrenceIndex=function(){var e=this.RULE_OCCURRENCE_STACK;return e[e.length-1]},r}();oy.TreeBuilder=DIe});var sq=w(ay=>{"use strict";Object.defineProperty(ay,"__esModule",{value:!0});ay.LexerAdapter=void 0;var nq=Hn(),kIe=function(){function r(){}return r.prototype.initLexerAdapter=function(){this.tokVector=[],this.tokVectorLength=0,this.currIdx=-1},Object.defineProperty(r.prototype,"input",{get:function(){return this.tokVector},set:function(e){if(this.selfAnalysisDone!==!0)throw Error("Missing invocation at the end of the Parser's constructor.");this.reset(),this.tokVector=e,this.tokVectorLength=e.length},enumerable:!1,configurable:!0}),r.prototype.SKIP_TOKEN=function(){return this.currIdx<=this.tokVector.length-2?(this.consumeToken(),this.LA(1)):nq.END_OF_FILE},r.prototype.LA=function(e){var t=this.currIdx+e;return t<0||this.tokVectorLength<=t?nq.END_OF_FILE:this.tokVector[t]},r.prototype.consumeToken=function(){this.currIdx++},r.prototype.exportLexerState=function(){return this.currIdx},r.prototype.importLexerState=function(e){this.currIdx=e},r.prototype.resetLexerState=function(){this.currIdx=-1},r.prototype.moveToTerminatedState=function(){this.currIdx=this.tokVector.length-1},r.prototype.getLexerPosition=function(){return this.exportLexerState()},r}();ay.LexerAdapter=kIe});var aq=w(Ay=>{"use strict";Object.defineProperty(Ay,"__esModule",{value:!0});Ay.RecognizerApi=void 0;var oq=Gt(),RIe=Wg(),Wv=Hn(),FIe=Cd(),NIe=Hv(),LIe=dn(),TIe=function(){function r(){}return r.prototype.ACTION=function(e){return e.call(this)},r.prototype.consume=function(e,t,i){return this.consumeInternal(t,e,i)},r.prototype.subrule=function(e,t,i){return this.subruleInternal(t,e,i)},r.prototype.option=function(e,t){return this.optionInternal(t,e)},r.prototype.or=function(e,t){return this.orInternal(t,e)},r.prototype.many=function(e,t){return this.manyInternal(e,t)},r.prototype.atLeastOne=function(e,t){return this.atLeastOneInternal(e,t)},r.prototype.CONSUME=function(e,t){return this.consumeInternal(e,0,t)},r.prototype.CONSUME1=function(e,t){return this.consumeInternal(e,1,t)},r.prototype.CONSUME2=function(e,t){return this.consumeInternal(e,2,t)},r.prototype.CONSUME3=function(e,t){return this.consumeInternal(e,3,t)},r.prototype.CONSUME4=function(e,t){return this.consumeInternal(e,4,t)},r.prototype.CONSUME5=function(e,t){return this.consumeInternal(e,5,t)},r.prototype.CONSUME6=function(e,t){return this.consumeInternal(e,6,t)},r.prototype.CONSUME7=function(e,t){return this.consumeInternal(e,7,t)},r.prototype.CONSUME8=function(e,t){return this.consumeInternal(e,8,t)},r.prototype.CONSUME9=function(e,t){return this.consumeInternal(e,9,t)},r.prototype.SUBRULE=function(e,t){return this.subruleInternal(e,0,t)},r.prototype.SUBRULE1=function(e,t){return this.subruleInternal(e,1,t)},r.prototype.SUBRULE2=function(e,t){return this.subruleInternal(e,2,t)},r.prototype.SUBRULE3=function(e,t){return this.subruleInternal(e,3,t)},r.prototype.SUBRULE4=function(e,t){return this.subruleInternal(e,4,t)},r.prototype.SUBRULE5=function(e,t){return this.subruleInternal(e,5,t)},r.prototype.SUBRULE6=function(e,t){return this.subruleInternal(e,6,t)},r.prototype.SUBRULE7=function(e,t){return this.subruleInternal(e,7,t)},r.prototype.SUBRULE8=function(e,t){return this.subruleInternal(e,8,t)},r.prototype.SUBRULE9=function(e,t){return this.subruleInternal(e,9,t)},r.prototype.OPTION=function(e){return this.optionInternal(e,0)},r.prototype.OPTION1=function(e){return this.optionInternal(e,1)},r.prototype.OPTION2=function(e){return this.optionInternal(e,2)},r.prototype.OPTION3=function(e){return this.optionInternal(e,3)},r.prototype.OPTION4=function(e){return this.optionInternal(e,4)},r.prototype.OPTION5=function(e){return this.optionInternal(e,5)},r.prototype.OPTION6=function(e){return this.optionInternal(e,6)},r.prototype.OPTION7=function(e){return this.optionInternal(e,7)},r.prototype.OPTION8=function(e){return this.optionInternal(e,8)},r.prototype.OPTION9=function(e){return this.optionInternal(e,9)},r.prototype.OR=function(e){return this.orInternal(e,0)},r.prototype.OR1=function(e){return this.orInternal(e,1)},r.prototype.OR2=function(e){return this.orInternal(e,2)},r.prototype.OR3=function(e){return this.orInternal(e,3)},r.prototype.OR4=function(e){return this.orInternal(e,4)},r.prototype.OR5=function(e){return this.orInternal(e,5)},r.prototype.OR6=function(e){return this.orInternal(e,6)},r.prototype.OR7=function(e){return this.orInternal(e,7)},r.prototype.OR8=function(e){return this.orInternal(e,8)},r.prototype.OR9=function(e){return this.orInternal(e,9)},r.prototype.MANY=function(e){this.manyInternal(0,e)},r.prototype.MANY1=function(e){this.manyInternal(1,e)},r.prototype.MANY2=function(e){this.manyInternal(2,e)},r.prototype.MANY3=function(e){this.manyInternal(3,e)},r.prototype.MANY4=function(e){this.manyInternal(4,e)},r.prototype.MANY5=function(e){this.manyInternal(5,e)},r.prototype.MANY6=function(e){this.manyInternal(6,e)},r.prototype.MANY7=function(e){this.manyInternal(7,e)},r.prototype.MANY8=function(e){this.manyInternal(8,e)},r.prototype.MANY9=function(e){this.manyInternal(9,e)},r.prototype.MANY_SEP=function(e){this.manySepFirstInternal(0,e)},r.prototype.MANY_SEP1=function(e){this.manySepFirstInternal(1,e)},r.prototype.MANY_SEP2=function(e){this.manySepFirstInternal(2,e)},r.prototype.MANY_SEP3=function(e){this.manySepFirstInternal(3,e)},r.prototype.MANY_SEP4=function(e){this.manySepFirstInternal(4,e)},r.prototype.MANY_SEP5=function(e){this.manySepFirstInternal(5,e)},r.prototype.MANY_SEP6=function(e){this.manySepFirstInternal(6,e)},r.prototype.MANY_SEP7=function(e){this.manySepFirstInternal(7,e)},r.prototype.MANY_SEP8=function(e){this.manySepFirstInternal(8,e)},r.prototype.MANY_SEP9=function(e){this.manySepFirstInternal(9,e)},r.prototype.AT_LEAST_ONE=function(e){this.atLeastOneInternal(0,e)},r.prototype.AT_LEAST_ONE1=function(e){return this.atLeastOneInternal(1,e)},r.prototype.AT_LEAST_ONE2=function(e){this.atLeastOneInternal(2,e)},r.prototype.AT_LEAST_ONE3=function(e){this.atLeastOneInternal(3,e)},r.prototype.AT_LEAST_ONE4=function(e){this.atLeastOneInternal(4,e)},r.prototype.AT_LEAST_ONE5=function(e){this.atLeastOneInternal(5,e)},r.prototype.AT_LEAST_ONE6=function(e){this.atLeastOneInternal(6,e)},r.prototype.AT_LEAST_ONE7=function(e){this.atLeastOneInternal(7,e)},r.prototype.AT_LEAST_ONE8=function(e){this.atLeastOneInternal(8,e)},r.prototype.AT_LEAST_ONE9=function(e){this.atLeastOneInternal(9,e)},r.prototype.AT_LEAST_ONE_SEP=function(e){this.atLeastOneSepFirstInternal(0,e)},r.prototype.AT_LEAST_ONE_SEP1=function(e){this.atLeastOneSepFirstInternal(1,e)},r.prototype.AT_LEAST_ONE_SEP2=function(e){this.atLeastOneSepFirstInternal(2,e)},r.prototype.AT_LEAST_ONE_SEP3=function(e){this.atLeastOneSepFirstInternal(3,e)},r.prototype.AT_LEAST_ONE_SEP4=function(e){this.atLeastOneSepFirstInternal(4,e)},r.prototype.AT_LEAST_ONE_SEP5=function(e){this.atLeastOneSepFirstInternal(5,e)},r.prototype.AT_LEAST_ONE_SEP6=function(e){this.atLeastOneSepFirstInternal(6,e)},r.prototype.AT_LEAST_ONE_SEP7=function(e){this.atLeastOneSepFirstInternal(7,e)},r.prototype.AT_LEAST_ONE_SEP8=function(e){this.atLeastOneSepFirstInternal(8,e)},r.prototype.AT_LEAST_ONE_SEP9=function(e){this.atLeastOneSepFirstInternal(9,e)},r.prototype.RULE=function(e,t,i){if(i===void 0&&(i=Wv.DEFAULT_RULE_CONFIG),(0,oq.contains)(this.definedRulesNames,e)){var n=FIe.defaultGrammarValidatorErrorProvider.buildDuplicateRuleNameError({topLevelRule:e,grammarName:this.className}),s={message:n,type:Wv.ParserDefinitionErrorType.DUPLICATE_RULE_NAME,ruleName:e};this.definitionErrors.push(s)}this.definedRulesNames.push(e);var o=this.defineRule(e,t,i);return this[e]=o,o},r.prototype.OVERRIDE_RULE=function(e,t,i){i===void 0&&(i=Wv.DEFAULT_RULE_CONFIG);var n=[];n=n.concat((0,NIe.validateRuleIsOverridden)(e,this.definedRulesNames,this.className)),this.definitionErrors=this.definitionErrors.concat(n);var s=this.defineRule(e,t,i);return this[e]=s,s},r.prototype.BACKTRACK=function(e,t){return function(){this.isBackTrackingStack.push(1);var i=this.saveRecogState();try{return e.apply(this,t),!0}catch(n){if((0,RIe.isRecognitionException)(n))return!1;throw n}finally{this.reloadRecogState(i),this.isBackTrackingStack.pop()}}},r.prototype.getGAstProductions=function(){return this.gastProductionsCache},r.prototype.getSerializedGastProductions=function(){return(0,LIe.serializeGrammar)((0,oq.values)(this.gastProductionsCache))},r}();Ay.RecognizerApi=TIe});var uq=w(cy=>{"use strict";Object.defineProperty(cy,"__esModule",{value:!0});cy.RecognizerEngine=void 0;var Pr=Gt(),Gn=ny(),ly=Wg(),Aq=Id(),Vg=Ed(),lq=Hn(),OIe=jv(),cq=SA(),Qd=Gg(),MIe=qv(),KIe=function(){function r(){}return r.prototype.initRecognizerEngine=function(e,t){if(this.className=(0,MIe.classNameFromInstance)(this),this.shortRuleNameToFull={},this.fullRuleNameToShort={},this.ruleShortNameIdx=256,this.tokenMatcher=Qd.tokenStructuredMatcherNoCategories,this.definedRulesNames=[],this.tokensMap={},this.isBackTrackingStack=[],this.RULE_STACK=[],this.RULE_OCCURRENCE_STACK=[],this.gastProductionsCache={},(0,Pr.has)(t,"serializedGrammar"))throw Error(`The Parser's configuration can no longer contain a property. - See: https://chevrotain.io/docs/changes/BREAKING_CHANGES.html#_6-0-0 - For Further details.`);if((0,Pr.isArray)(e)){if((0,Pr.isEmpty)(e))throw Error(`A Token Vocabulary cannot be empty. - Note that the first argument for the parser constructor - is no longer a Token vector (since v4.0).`);if(typeof e[0].startOffset=="number")throw Error(`The Parser constructor no longer accepts a token vector as the first argument. - See: https://chevrotain.io/docs/changes/BREAKING_CHANGES.html#_4-0-0 - For Further details.`)}if((0,Pr.isArray)(e))this.tokensMap=(0,Pr.reduce)(e,function(o,a){return o[a.name]=a,o},{});else if((0,Pr.has)(e,"modes")&&(0,Pr.every)((0,Pr.flatten)((0,Pr.values)(e.modes)),Qd.isTokenType)){var i=(0,Pr.flatten)((0,Pr.values)(e.modes)),n=(0,Pr.uniq)(i);this.tokensMap=(0,Pr.reduce)(n,function(o,a){return o[a.name]=a,o},{})}else if((0,Pr.isObject)(e))this.tokensMap=(0,Pr.cloneObj)(e);else throw new Error(" argument must be An Array of Token constructors, A dictionary of Token constructors or an IMultiModeLexerDefinition");this.tokensMap.EOF=cq.EOF;var s=(0,Pr.every)((0,Pr.values)(e),function(o){return(0,Pr.isEmpty)(o.categoryMatches)});this.tokenMatcher=s?Qd.tokenStructuredMatcherNoCategories:Qd.tokenStructuredMatcher,(0,Qd.augmentTokenTypes)((0,Pr.values)(this.tokensMap))},r.prototype.defineRule=function(e,t,i){if(this.selfAnalysisDone)throw Error("Grammar rule <"+e+`> may not be defined after the 'performSelfAnalysis' method has been called' -Make sure that all grammar rule definitions are done before 'performSelfAnalysis' is called.`);var n=(0,Pr.has)(i,"resyncEnabled")?i.resyncEnabled:lq.DEFAULT_RULE_CONFIG.resyncEnabled,s=(0,Pr.has)(i,"recoveryValueFunc")?i.recoveryValueFunc:lq.DEFAULT_RULE_CONFIG.recoveryValueFunc,o=this.ruleShortNameIdx<t},r.prototype.orInternal=function(e,t){var i=this.getKeyForAutomaticLookahead(Gn.OR_IDX,t),n=(0,Pr.isArray)(e)?e:e.DEF,s=this.getLaFuncFromCache(i),o=s.call(this,n);if(o!==void 0){var a=n[o];return a.ALT.call(this)}this.raiseNoAltException(t,e.ERR_MSG)},r.prototype.ruleFinallyStateUpdate=function(){if(this.RULE_STACK.pop(),this.RULE_OCCURRENCE_STACK.pop(),this.cstFinallyStateUpdate(),this.RULE_STACK.length===0&&this.isAtEndOfInput()===!1){var e=this.LA(1),t=this.errorMessageProvider.buildNotAllInputParsedMessage({firstRedundant:e,ruleName:this.getCurrRuleFullName()});this.SAVE_ERROR(new ly.NotAllInputParsedException(t,e))}},r.prototype.subruleInternal=function(e,t,i){var n;try{var s=i!==void 0?i.ARGS:void 0;return n=e.call(this,t,s),this.cstPostNonTerminal(n,i!==void 0&&i.LABEL!==void 0?i.LABEL:e.ruleName),n}catch(o){this.subruleInternalError(o,i,e.ruleName)}},r.prototype.subruleInternalError=function(e,t,i){throw(0,ly.isRecognitionException)(e)&&e.partialCstResult!==void 0&&(this.cstPostNonTerminal(e.partialCstResult,t!==void 0&&t.LABEL!==void 0?t.LABEL:i),delete e.partialCstResult),e},r.prototype.consumeInternal=function(e,t,i){var n;try{var s=this.LA(1);this.tokenMatcher(s,e)===!0?(this.consumeToken(),n=s):this.consumeInternalError(e,s,i)}catch(o){n=this.consumeInternalRecovery(e,t,o)}return this.cstPostTerminal(i!==void 0&&i.LABEL!==void 0?i.LABEL:e.name,n),n},r.prototype.consumeInternalError=function(e,t,i){var n,s=this.LA(0);throw i!==void 0&&i.ERR_MSG?n=i.ERR_MSG:n=this.errorMessageProvider.buildMismatchTokenMessage({expected:e,actual:t,previous:s,ruleName:this.getCurrRuleFullName()}),this.SAVE_ERROR(new ly.MismatchedTokenException(n,t,s))},r.prototype.consumeInternalRecovery=function(e,t,i){if(this.recoveryEnabled&&i.name==="MismatchedTokenException"&&!this.isBackTracking()){var n=this.getFollowsForInRuleRecovery(e,t);try{return this.tryInRuleRecovery(e,n)}catch(s){throw s.name===OIe.IN_RULE_RECOVERY_EXCEPTION?i:s}}else throw i},r.prototype.saveRecogState=function(){var e=this.errors,t=(0,Pr.cloneArr)(this.RULE_STACK);return{errors:e,lexerState:this.exportLexerState(),RULE_STACK:t,CST_STACK:this.CST_STACK}},r.prototype.reloadRecogState=function(e){this.errors=e.errors,this.importLexerState(e.lexerState),this.RULE_STACK=e.RULE_STACK},r.prototype.ruleInvocationStateUpdate=function(e,t,i){this.RULE_OCCURRENCE_STACK.push(i),this.RULE_STACK.push(e),this.cstInvocationStateUpdate(t,e)},r.prototype.isBackTracking=function(){return this.isBackTrackingStack.length!==0},r.prototype.getCurrRuleFullName=function(){var e=this.getLastExplicitRuleShortName();return this.shortRuleNameToFull[e]},r.prototype.shortRuleNameToFullName=function(e){return this.shortRuleNameToFull[e]},r.prototype.isAtEndOfInput=function(){return this.tokenMatcher(this.LA(1),cq.EOF)},r.prototype.reset=function(){this.resetLexerState(),this.isBackTrackingStack=[],this.errors=[],this.RULE_STACK=[],this.CST_STACK=[],this.RULE_OCCURRENCE_STACK=[]},r}();cy.RecognizerEngine=KIe});var fq=w(uy=>{"use strict";Object.defineProperty(uy,"__esModule",{value:!0});uy.ErrorHandler=void 0;var zv=Wg(),Vv=Gt(),gq=Id(),UIe=Hn(),HIe=function(){function r(){}return r.prototype.initErrorHandler=function(e){this._errors=[],this.errorMessageProvider=(0,Vv.has)(e,"errorMessageProvider")?e.errorMessageProvider:UIe.DEFAULT_PARSER_CONFIG.errorMessageProvider},r.prototype.SAVE_ERROR=function(e){if((0,zv.isRecognitionException)(e))return e.context={ruleStack:this.getHumanReadableRuleStack(),ruleOccurrenceStack:(0,Vv.cloneArr)(this.RULE_OCCURRENCE_STACK)},this._errors.push(e),e;throw Error("Trying to save an Error which is not a RecognitionException")},Object.defineProperty(r.prototype,"errors",{get:function(){return(0,Vv.cloneArr)(this._errors)},set:function(e){this._errors=e},enumerable:!1,configurable:!0}),r.prototype.raiseEarlyExitException=function(e,t,i){for(var n=this.getCurrRuleFullName(),s=this.getGAstProductions()[n],o=(0,gq.getLookaheadPathsForOptionalProd)(e,s,t,this.maxLookahead),a=o[0],l=[],c=1;c<=this.maxLookahead;c++)l.push(this.LA(c));var u=this.errorMessageProvider.buildEarlyExitMessage({expectedIterationPaths:a,actual:l,previous:this.LA(0),customUserDescription:i,ruleName:n});throw this.SAVE_ERROR(new zv.EarlyExitException(u,this.LA(1),this.LA(0)))},r.prototype.raiseNoAltException=function(e,t){for(var i=this.getCurrRuleFullName(),n=this.getGAstProductions()[i],s=(0,gq.getLookaheadPathsForOr)(e,n,this.maxLookahead),o=[],a=1;a<=this.maxLookahead;a++)o.push(this.LA(a));var l=this.LA(0),c=this.errorMessageProvider.buildNoViableAltMessage({expectedPathsPerAlt:s,actual:o,previous:l,customUserDescription:t,ruleName:this.getCurrRuleFullName()});throw this.SAVE_ERROR(new zv.NoViableAltException(c,this.LA(1),l))},r}();uy.ErrorHandler=HIe});var dq=w(gy=>{"use strict";Object.defineProperty(gy,"__esModule",{value:!0});gy.ContentAssist=void 0;var hq=Ed(),pq=Gt(),GIe=function(){function r(){}return r.prototype.initContentAssist=function(){},r.prototype.computeContentAssist=function(e,t){var i=this.gastProductionsCache[e];if((0,pq.isUndefined)(i))throw Error("Rule ->"+e+"<- does not exist in this grammar.");return(0,hq.nextPossibleTokensAfter)([i],t,this.tokenMatcher,this.maxLookahead)},r.prototype.getNextPossibleTokenTypes=function(e){var t=(0,pq.first)(e.ruleStack),i=this.getGAstProductions(),n=i[t],s=new hq.NextAfterTokenWalker(n,e).startWalking();return s},r}();gy.ContentAssist=GIe});var Qq=w(py=>{"use strict";Object.defineProperty(py,"__esModule",{value:!0});py.GastRecorder=void 0;var En=Gt(),Ro=dn(),YIe=gd(),Iq=Gg(),yq=SA(),jIe=Hn(),qIe=ny(),hy={description:"This Object indicates the Parser is during Recording Phase"};Object.freeze(hy);var Cq=!0,mq=Math.pow(2,qIe.BITS_FOR_OCCURRENCE_IDX)-1,wq=(0,yq.createToken)({name:"RECORDING_PHASE_TOKEN",pattern:YIe.Lexer.NA});(0,Iq.augmentTokenTypes)([wq]);var Bq=(0,yq.createTokenInstance)(wq,`This IToken indicates the Parser is in Recording Phase - See: https://chevrotain.io/docs/guide/internals.html#grammar-recording for details`,-1,-1,-1,-1,-1,-1);Object.freeze(Bq);var JIe={name:`This CSTNode indicates the Parser is in Recording Phase - See: https://chevrotain.io/docs/guide/internals.html#grammar-recording for details`,children:{}},WIe=function(){function r(){}return r.prototype.initGastRecorder=function(e){this.recordingProdStack=[],this.RECORDING_PHASE=!1},r.prototype.enableRecording=function(){var e=this;this.RECORDING_PHASE=!0,this.TRACE_INIT("Enable Recording",function(){for(var t=function(n){var s=n>0?n:"";e["CONSUME"+s]=function(o,a){return this.consumeInternalRecord(o,n,a)},e["SUBRULE"+s]=function(o,a){return this.subruleInternalRecord(o,n,a)},e["OPTION"+s]=function(o){return this.optionInternalRecord(o,n)},e["OR"+s]=function(o){return this.orInternalRecord(o,n)},e["MANY"+s]=function(o){this.manyInternalRecord(n,o)},e["MANY_SEP"+s]=function(o){this.manySepFirstInternalRecord(n,o)},e["AT_LEAST_ONE"+s]=function(o){this.atLeastOneInternalRecord(n,o)},e["AT_LEAST_ONE_SEP"+s]=function(o){this.atLeastOneSepFirstInternalRecord(n,o)}},i=0;i<10;i++)t(i);e.consume=function(n,s,o){return this.consumeInternalRecord(s,n,o)},e.subrule=function(n,s,o){return this.subruleInternalRecord(s,n,o)},e.option=function(n,s){return this.optionInternalRecord(s,n)},e.or=function(n,s){return this.orInternalRecord(s,n)},e.many=function(n,s){this.manyInternalRecord(n,s)},e.atLeastOne=function(n,s){this.atLeastOneInternalRecord(n,s)},e.ACTION=e.ACTION_RECORD,e.BACKTRACK=e.BACKTRACK_RECORD,e.LA=e.LA_RECORD})},r.prototype.disableRecording=function(){var e=this;this.RECORDING_PHASE=!1,this.TRACE_INIT("Deleting Recording methods",function(){for(var t=0;t<10;t++){var i=t>0?t:"";delete e["CONSUME"+i],delete e["SUBRULE"+i],delete e["OPTION"+i],delete e["OR"+i],delete e["MANY"+i],delete e["MANY_SEP"+i],delete e["AT_LEAST_ONE"+i],delete e["AT_LEAST_ONE_SEP"+i]}delete e.consume,delete e.subrule,delete e.option,delete e.or,delete e.many,delete e.atLeastOne,delete e.ACTION,delete e.BACKTRACK,delete e.LA})},r.prototype.ACTION_RECORD=function(e){},r.prototype.BACKTRACK_RECORD=function(e,t){return function(){return!0}},r.prototype.LA_RECORD=function(e){return jIe.END_OF_FILE},r.prototype.topLevelRuleRecord=function(e,t){try{var i=new Ro.Rule({definition:[],name:e});return i.name=e,this.recordingProdStack.push(i),t.call(this),this.recordingProdStack.pop(),i}catch(n){if(n.KNOWN_RECORDER_ERROR!==!0)try{n.message=n.message+` - This error was thrown during the "grammar recording phase" For more info see: - https://chevrotain.io/docs/guide/internals.html#grammar-recording`}catch{throw n}throw n}},r.prototype.optionInternalRecord=function(e,t){return bd.call(this,Ro.Option,e,t)},r.prototype.atLeastOneInternalRecord=function(e,t){bd.call(this,Ro.RepetitionMandatory,t,e)},r.prototype.atLeastOneSepFirstInternalRecord=function(e,t){bd.call(this,Ro.RepetitionMandatoryWithSeparator,t,e,Cq)},r.prototype.manyInternalRecord=function(e,t){bd.call(this,Ro.Repetition,t,e)},r.prototype.manySepFirstInternalRecord=function(e,t){bd.call(this,Ro.RepetitionWithSeparator,t,e,Cq)},r.prototype.orInternalRecord=function(e,t){return zIe.call(this,e,t)},r.prototype.subruleInternalRecord=function(e,t,i){if(fy(t),!e||(0,En.has)(e,"ruleName")===!1){var n=new Error(" argument is invalid"+(" expecting a Parser method reference but got: <"+JSON.stringify(e)+">")+(` - inside top level rule: <`+this.recordingProdStack[0].name+">"));throw n.KNOWN_RECORDER_ERROR=!0,n}var s=(0,En.peek)(this.recordingProdStack),o=e.ruleName,a=new Ro.NonTerminal({idx:t,nonTerminalName:o,label:i==null?void 0:i.LABEL,referencedRule:void 0});return s.definition.push(a),this.outputCst?JIe:hy},r.prototype.consumeInternalRecord=function(e,t,i){if(fy(t),!(0,Iq.hasShortKeyProperty)(e)){var n=new Error(" argument is invalid"+(" expecting a TokenType reference but got: <"+JSON.stringify(e)+">")+(` - inside top level rule: <`+this.recordingProdStack[0].name+">"));throw n.KNOWN_RECORDER_ERROR=!0,n}var s=(0,En.peek)(this.recordingProdStack),o=new Ro.Terminal({idx:t,terminalType:e,label:i==null?void 0:i.LABEL});return s.definition.push(o),Bq},r}();py.GastRecorder=WIe;function bd(r,e,t,i){i===void 0&&(i=!1),fy(t);var n=(0,En.peek)(this.recordingProdStack),s=(0,En.isFunction)(e)?e:e.DEF,o=new r({definition:[],idx:t});return i&&(o.separator=e.SEP),(0,En.has)(e,"MAX_LOOKAHEAD")&&(o.maxLookahead=e.MAX_LOOKAHEAD),this.recordingProdStack.push(o),s.call(this),n.definition.push(o),this.recordingProdStack.pop(),hy}function zIe(r,e){var t=this;fy(e);var i=(0,En.peek)(this.recordingProdStack),n=(0,En.isArray)(r)===!1,s=n===!1?r:r.DEF,o=new Ro.Alternation({definition:[],idx:e,ignoreAmbiguities:n&&r.IGNORE_AMBIGUITIES===!0});(0,En.has)(r,"MAX_LOOKAHEAD")&&(o.maxLookahead=r.MAX_LOOKAHEAD);var a=(0,En.some)(s,function(l){return(0,En.isFunction)(l.GATE)});return o.hasPredicates=a,i.definition.push(o),(0,En.forEach)(s,function(l){var c=new Ro.Alternative({definition:[]});o.definition.push(c),(0,En.has)(l,"IGNORE_AMBIGUITIES")?c.ignoreAmbiguities=l.IGNORE_AMBIGUITIES:(0,En.has)(l,"GATE")&&(c.ignoreAmbiguities=!0),t.recordingProdStack.push(c),l.ALT.call(t),t.recordingProdStack.pop()}),hy}function Eq(r){return r===0?"":""+r}function fy(r){if(r<0||r>mq){var e=new Error("Invalid DSL Method idx value: <"+r+`> - `+("Idx value must be a none negative value smaller than "+(mq+1)));throw e.KNOWN_RECORDER_ERROR=!0,e}}});var Sq=w(dy=>{"use strict";Object.defineProperty(dy,"__esModule",{value:!0});dy.PerformanceTracer=void 0;var bq=Gt(),VIe=Hn(),XIe=function(){function r(){}return r.prototype.initPerformanceTracer=function(e){if((0,bq.has)(e,"traceInitPerf")){var t=e.traceInitPerf,i=typeof t=="number";this.traceInitMaxIdent=i?t:1/0,this.traceInitPerf=i?t>0:t}else this.traceInitMaxIdent=0,this.traceInitPerf=VIe.DEFAULT_PARSER_CONFIG.traceInitPerf;this.traceInitIndent=-1},r.prototype.TRACE_INIT=function(e,t){if(this.traceInitPerf===!0){this.traceInitIndent++;var i=new Array(this.traceInitIndent+1).join(" ");this.traceInitIndent <"+e+">");var n=(0,bq.timer)(t),s=n.time,o=n.value,a=s>10?console.warn:console.log;return this.traceInitIndent time: "+s+"ms"),this.traceInitIndent--,o}else return t()},r}();dy.PerformanceTracer=XIe});var vq=w(Cy=>{"use strict";Object.defineProperty(Cy,"__esModule",{value:!0});Cy.applyMixins=void 0;function _Ie(r,e){e.forEach(function(t){var i=t.prototype;Object.getOwnPropertyNames(i).forEach(function(n){if(n!=="constructor"){var s=Object.getOwnPropertyDescriptor(i,n);s&&(s.get||s.set)?Object.defineProperty(r.prototype,n,s):r.prototype[n]=t.prototype[n]}})})}Cy.applyMixins=_Ie});var Hn=w(dr=>{"use strict";var Dq=dr&&dr.__extends||function(){var r=function(e,t){return r=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(i,n){i.__proto__=n}||function(i,n){for(var s in n)Object.prototype.hasOwnProperty.call(n,s)&&(i[s]=n[s])},r(e,t)};return function(e,t){if(typeof t!="function"&&t!==null)throw new TypeError("Class extends value "+String(t)+" is not a constructor or null");r(e,t);function i(){this.constructor=e}e.prototype=t===null?Object.create(t):(i.prototype=t.prototype,new i)}}();Object.defineProperty(dr,"__esModule",{value:!0});dr.EmbeddedActionsParser=dr.CstParser=dr.Parser=dr.EMPTY_ALT=dr.ParserDefinitionErrorType=dr.DEFAULT_RULE_CONFIG=dr.DEFAULT_PARSER_CONFIG=dr.END_OF_FILE=void 0;var Xi=Gt(),ZIe=fj(),xq=SA(),kq=Cd(),Pq=Kj(),$Ie=jv(),eye=Wj(),tye=iq(),rye=sq(),iye=aq(),nye=uq(),sye=fq(),oye=dq(),aye=Qq(),Aye=Sq(),lye=vq();dr.END_OF_FILE=(0,xq.createTokenInstance)(xq.EOF,"",NaN,NaN,NaN,NaN,NaN,NaN);Object.freeze(dr.END_OF_FILE);dr.DEFAULT_PARSER_CONFIG=Object.freeze({recoveryEnabled:!1,maxLookahead:3,dynamicTokensEnabled:!1,outputCst:!0,errorMessageProvider:kq.defaultParserErrorProvider,nodeLocationTracking:"none",traceInitPerf:!1,skipValidations:!1});dr.DEFAULT_RULE_CONFIG=Object.freeze({recoveryValueFunc:function(){},resyncEnabled:!0});var cye;(function(r){r[r.INVALID_RULE_NAME=0]="INVALID_RULE_NAME",r[r.DUPLICATE_RULE_NAME=1]="DUPLICATE_RULE_NAME",r[r.INVALID_RULE_OVERRIDE=2]="INVALID_RULE_OVERRIDE",r[r.DUPLICATE_PRODUCTIONS=3]="DUPLICATE_PRODUCTIONS",r[r.UNRESOLVED_SUBRULE_REF=4]="UNRESOLVED_SUBRULE_REF",r[r.LEFT_RECURSION=5]="LEFT_RECURSION",r[r.NONE_LAST_EMPTY_ALT=6]="NONE_LAST_EMPTY_ALT",r[r.AMBIGUOUS_ALTS=7]="AMBIGUOUS_ALTS",r[r.CONFLICT_TOKENS_RULES_NAMESPACE=8]="CONFLICT_TOKENS_RULES_NAMESPACE",r[r.INVALID_TOKEN_NAME=9]="INVALID_TOKEN_NAME",r[r.NO_NON_EMPTY_LOOKAHEAD=10]="NO_NON_EMPTY_LOOKAHEAD",r[r.AMBIGUOUS_PREFIX_ALTS=11]="AMBIGUOUS_PREFIX_ALTS",r[r.TOO_MANY_ALTS=12]="TOO_MANY_ALTS"})(cye=dr.ParserDefinitionErrorType||(dr.ParserDefinitionErrorType={}));function uye(r){return r===void 0&&(r=void 0),function(){return r}}dr.EMPTY_ALT=uye;var my=function(){function r(e,t){this.definitionErrors=[],this.selfAnalysisDone=!1;var i=this;if(i.initErrorHandler(t),i.initLexerAdapter(),i.initLooksAhead(t),i.initRecognizerEngine(e,t),i.initRecoverable(t),i.initTreeBuilder(t),i.initContentAssist(),i.initGastRecorder(t),i.initPerformanceTracer(t),(0,Xi.has)(t,"ignoredIssues"))throw new Error(`The IParserConfig property has been deprecated. - Please use the flag on the relevant DSL method instead. - See: https://chevrotain.io/docs/guide/resolving_grammar_errors.html#IGNORING_AMBIGUITIES - For further details.`);this.skipValidations=(0,Xi.has)(t,"skipValidations")?t.skipValidations:dr.DEFAULT_PARSER_CONFIG.skipValidations}return r.performSelfAnalysis=function(e){throw Error("The **static** `performSelfAnalysis` method has been deprecated. \nUse the **instance** method with the same name instead.")},r.prototype.performSelfAnalysis=function(){var e=this;this.TRACE_INIT("performSelfAnalysis",function(){var t;e.selfAnalysisDone=!0;var i=e.className;e.TRACE_INIT("toFastProps",function(){(0,Xi.toFastProperties)(e)}),e.TRACE_INIT("Grammar Recording",function(){try{e.enableRecording(),(0,Xi.forEach)(e.definedRulesNames,function(s){var o=e[s],a=o.originalGrammarAction,l=void 0;e.TRACE_INIT(s+" Rule",function(){l=e.topLevelRuleRecord(s,a)}),e.gastProductionsCache[s]=l})}finally{e.disableRecording()}});var n=[];if(e.TRACE_INIT("Grammar Resolving",function(){n=(0,Pq.resolveGrammar)({rules:(0,Xi.values)(e.gastProductionsCache)}),e.definitionErrors=e.definitionErrors.concat(n)}),e.TRACE_INIT("Grammar Validations",function(){if((0,Xi.isEmpty)(n)&&e.skipValidations===!1){var s=(0,Pq.validateGrammar)({rules:(0,Xi.values)(e.gastProductionsCache),maxLookahead:e.maxLookahead,tokenTypes:(0,Xi.values)(e.tokensMap),errMsgProvider:kq.defaultGrammarValidatorErrorProvider,grammarName:i});e.definitionErrors=e.definitionErrors.concat(s)}}),(0,Xi.isEmpty)(e.definitionErrors)&&(e.recoveryEnabled&&e.TRACE_INIT("computeAllProdsFollows",function(){var s=(0,ZIe.computeAllProdsFollows)((0,Xi.values)(e.gastProductionsCache));e.resyncFollows=s}),e.TRACE_INIT("ComputeLookaheadFunctions",function(){e.preComputeLookaheadFunctions((0,Xi.values)(e.gastProductionsCache))})),!r.DEFER_DEFINITION_ERRORS_HANDLING&&!(0,Xi.isEmpty)(e.definitionErrors))throw t=(0,Xi.map)(e.definitionErrors,function(s){return s.message}),new Error(`Parser Definition Errors detected: - `+t.join(` -------------------------------- -`))})},r.DEFER_DEFINITION_ERRORS_HANDLING=!1,r}();dr.Parser=my;(0,lye.applyMixins)(my,[$Ie.Recoverable,eye.LooksAhead,tye.TreeBuilder,rye.LexerAdapter,nye.RecognizerEngine,iye.RecognizerApi,sye.ErrorHandler,oye.ContentAssist,aye.GastRecorder,Aye.PerformanceTracer]);var gye=function(r){Dq(e,r);function e(t,i){i===void 0&&(i=dr.DEFAULT_PARSER_CONFIG);var n=this,s=(0,Xi.cloneObj)(i);return s.outputCst=!0,n=r.call(this,t,s)||this,n}return e}(my);dr.CstParser=gye;var fye=function(r){Dq(e,r);function e(t,i){i===void 0&&(i=dr.DEFAULT_PARSER_CONFIG);var n=this,s=(0,Xi.cloneObj)(i);return s.outputCst=!1,n=r.call(this,t,s)||this,n}return e}(my);dr.EmbeddedActionsParser=fye});var Fq=w(Ey=>{"use strict";Object.defineProperty(Ey,"__esModule",{value:!0});Ey.createSyntaxDiagramsCode=void 0;var Rq=pv();function hye(r,e){var t=e===void 0?{}:e,i=t.resourceBase,n=i===void 0?"https://unpkg.com/chevrotain@"+Rq.VERSION+"/diagrams/":i,s=t.css,o=s===void 0?"https://unpkg.com/chevrotain@"+Rq.VERSION+"/diagrams/diagrams.css":s,a=` - - - - - -`,l=` - -`,c=` -