diff --git a/.gitmodules b/.gitmodules index 82eee0e..9653555 100644 --- a/.gitmodules +++ b/.gitmodules @@ -11,3 +11,6 @@ path = docs/pingcap-docs/docs url = git@github.com:pingcap/docs.git branch = release-8.5 +[submodule "ref/fs"] + path = ref/fs + url = git@github.com:tidbcloud/fs.git diff --git a/AGENTS.md b/AGENTS.md index 14934dd..d5001f6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -92,6 +92,7 @@ Implemented: - `ti db format-db-connection-string` - `ti db execute-sql-statement` - `ti fs create-file-system` +- `ti fs import-file-system-token` - `ti fs delete-file-system` - `ti fs list-file-systems` - `ti fs describe-file-system` @@ -141,7 +142,9 @@ Implemented: - structured JSON/text rendering and JMESPath `--query` - `--dry-run` on mutating control-plane commands - TiDB Cloud Digest-auth API client foundation and auth/authz error mapping -- profile-scoped 1:N ti fs resource registry with per-resource credentials +- region-scoped remote ti fs inventory, profile-scoped ID-keyed credentials, + and legacy credential migration from + `docs/spec/done/0028-remote-fs-resource-inventory.md` - ti fs/fs-git/fs-journal/fs-vault commands routed through the bundled `ti-drive9` companion, with ti-owned profile loading, credential storage, region resolution, and output/error handling @@ -157,6 +160,12 @@ There are no registered placeholder commands at the current stage. Implemented mutating commands support `--dry-run` where their command contract declares dry-run support. +The completed remote ti fs inventory implementation and its verified regional +rollout status are recorded in +`docs/spec/done/0028-remote-fs-resource-inventory.md`. Future Drive9 region +publication and cleanup of historical backend tenant bindings are external +deployment work and do not reopen the ti client spec. + ## Reference Code - `ref/tidbcloud-cli/` is the previous TiDB Cloud CLI implementation. Use it as @@ -165,10 +174,11 @@ dry-run support. - `ref/drive9/` is the filesystem reference implementation. Use it as context for filesystem commands, mount behavior, and data-plane semantics. In ti user-facing output, this domain is always called `ti fs`. +- `ref/fs/` is the TiDB Filesystem server deployed for the Drive9-backed TiDB + Cloud Filesystem service. Use it to verify server routes, TiDB Cloud IAM and + billing authorization, tenant inventory and lifecycle behavior, quotas, and + data-plane contracts. It is server reference code, not a ti dependency. - `ref/serverless-js/` is a reference for the HTTPS SQL API call shape. -- `ref/fs/` is the TiDB Cloud deployment wrapper around Drive9. Use it only to - understand the hosted filesystem API and authorization boundary. - Reference directories are not product source for ti. They exist only to give agents context and implementation examples. In main project code, behave as if `ref/` does not exist: @@ -253,17 +263,12 @@ vault grant reads, vault mount read on macOS/Linux hosts when available, journal create/append/read/search/verify, public Git clone/hydrate/worktree flows, mount and drain through the companion runtime, and explicit WebDAV fallback when the platform supports it. -If the live profile has no registry resource named by `TI_LIVE_FS_NAME` or -`workspace`, the suite creates that temporary ti fs resource, stores its -metadata and API key in the profile-scoped resource registry, and deletes only -that auto-created resource before the DB lifecycle needs the Starter slot, or -when the test process exits if execution stops earlier. -The live suite also attempts a separate 1:N registry lifecycle with two unique -`ti-e2e-fs-*` resources, covering create, list, default selection, explicit -selection, isolated deletion, and cleanup. If the second resource is rejected -specifically because Starter quota is full, complete the single-resource live -flow and rely on `make e2e` for fake-companion multi-resource routing coverage. -Never delete a pre-existing resource to make room for this test. +If remote inventory has no resource with a local token, the suite creates one +temporary ti fs resource, records the server-assigned ID, and deletes only +that ID before the DB lifecycle needs the Starter slot or when the process +exits. `TI_LIVE_FS_ID` may select a remotely visible resource that already has +local credentials. Never delete a pre-existing resource to make room for a +test. Fake-companion e2e covers multiple remote resources and ID routing. When a service command is implemented, add its real live verification to `make live-e2e`; do not leave the target at profile, smoke-test-only, or mock-only coverage. @@ -317,7 +322,7 @@ internal/db/sqlsingle/ one-statement validation internal/db/validate/ DB flag and request validation helpers internal/dryrun/ shared dry-run result envelope internal/fs/ ti fs control-plane, data-plane, and mount use cases -internal/fs/fscred/ profile-scoped ti fs registry, selection, and migration +internal/fs/fscred/ ID-keyed ti fs credentials, selection, and legacy migration internal/fs/mountlocator/ non-secret Drive9 background mount routing state internal/oplog/ local JSONL operation log writer internal/output/ structured JSON/text/raw rendering @@ -336,7 +341,7 @@ docs/priciples.md product principles and MVP scope source of truth docs/spec/ pending requirement specs docs/spec/done/ completed requirement specs docs/pingcap-docs/docs/ pingcap/docs English documentation submodule -ref/ read-only reference implementations +ref/ read-only client and server reference implementations ``` Keep one package per directory. Package names should be short, lowercase, and @@ -424,7 +429,7 @@ Follow these rules unless `docs/priciples.md` is updated: add command-specific `--version ` flags; use names such as `--target-version` when a command needs a version input. - `ti update --check` and `ti update` use GitHub Releases metadata and - must not read or mutate `~/.ti/` or inspect/migrate `~/.tdc/`. + must not read or mutate `~/.ti/` or inspect/migrate `~/.ti/`. - `ti update` may replace only ti-owned archive/script installs. It must refuse local, unknown, Homebrew, Scoop, Winget, or other package-manager installs with actionable guidance. @@ -500,16 +505,16 @@ Implemented command behavior: - `ti db execute-sql-statement --db-cluster-id --admin --sql "select 1"` - `ti db execute-sql-statement --db-cluster-id --transport https --sql "select 1"` - `ti db execute-sql-statement --db-cluster-id --transport mysql --sql "select 1"` -- `ti fs create-file-system --file-system-name workspace` -- `ti fs create-file-system --file-system-name workspace --wait` -- `ti fs create-file-system --file-system-name workspace --dry-run` -- `ti fs create-file-system --file-system-name scratch` -- `ti fs delete-file-system --file-system-name workspace` -- `ti fs delete-file-system --file-system-name workspace --dry-run` +- `ti fs create-file-system` +- `ti fs create-file-system --wait` +- `ti fs create-file-system --dry-run` +- `ti fs import-file-system-token --from-file ./fs-token` +- `ti fs delete-file-system --file-system-id ` +- `ti fs delete-file-system --file-system-id --dry-run` - `ti fs list-file-systems` -- `ti fs describe-file-system --file-system-name workspace` +- `ti fs describe-file-system --file-system-id ` - `ti fs check-file-system` -- `ti fs check-file-system --file-system-name workspace` +- `ti fs check-file-system --file-system-id ` - `ti fs copy-file --from-local ./README.md --to-remote /workspace/README.md` - `ti fs copy-file --from-remote /workspace/README.md --to-local ./README.copy.md --create-parents` - `ti fs copy-file --from-remote /workspace/README.md --to-remote /workspace/README.copy.md` @@ -549,13 +554,13 @@ Implemented command behavior: - `ti fs pack-file-system --local-root ~/.ti/local/fs/demo --remote-root /workspace --mount-profile portable` - `ti fs pack-file-system --mount-path ./workspace` - `ti fs unpack-file-system --local-root ~/.ti/local/fs/demo --remote-root /workspace --mount-profile portable` -- `ti fs mount-file-system --file-system-name workspace --mount-path ./workspace` -- `ti fs mount-file-system --file-system-name workspace --mount-path ./workspace --driver fuse` -- `ti fs mount-file-system --file-system-name workspace --mount-path ./workspace --driver webdav` -- `ti fs mount-file-system --file-system-name workspace --mount-path ./workspace --mount-profile coding-agent` -- `ti fs mount-file-system --file-system-name workspace --mount-path ./workspace --mount-profile portable --pack-path /` -- `ti fs mount-file-system --file-system-name workspace --mount-path ./workspace --driver fuse --read-cache-size-mb 256 --read-cache-max-file-mb 16` -- `ti fs mount-file-system --file-system-name workspace --mount-path ./workspace --driver fuse --cache-dir ~/.ti/cache/workspace --write-back-cache=false` +- `ti fs mount-file-system --file-system-id --mount-path ./workspace` +- `ti fs mount-file-system --file-system-id --mount-path ./workspace --driver fuse` +- `ti fs mount-file-system --file-system-id --mount-path ./workspace --driver webdav` +- `ti fs mount-file-system --file-system-id --mount-path ./workspace --mount-profile coding-agent` +- `ti fs mount-file-system --file-system-id --mount-path ./workspace --mount-profile portable --pack-path /` +- `ti fs mount-file-system --file-system-id --mount-path ./workspace --driver fuse --read-cache-size-mb 256 --read-cache-max-file-mb 16` +- `ti fs mount-file-system --file-system-id --mount-path ./workspace --driver fuse --cache-dir ~/.ti/cache/workspace --write-back-cache=false` - `ti fs drain-file-system --mount-path ./workspace` - `ti fs drain-file-system --mount-path ./workspace --timeout 30s` - `ti fs unmount-file-system --mount-path ./workspace` @@ -602,6 +607,7 @@ Registered command surface: - `ti db format-db-connection-string` - `ti db execute-sql-statement` - `ti fs create-file-system` +- `ti fs import-file-system-token` - `ti fs delete-file-system` - `ti fs list-file-systems` - `ti fs describe-file-system` @@ -746,32 +752,30 @@ tidb_cloud_private_key = "..." Starter cluster. If it is absent and `--project-id` is not provided, the create request omits the project label and TiDB Cloud selects the account default. -One profile can own multiple ti fs resources. The main config stores neither a -default resource name nor resource credentials. +One profile can access multiple remotely inventoried ti fs resources. The main +config stores neither a default resource nor resource credentials. -Each resource stores metadata and credentials separately: +New local credentials are keyed by the server-assigned file system ID: ```text -~/.ti/fs_resources///config -~/.ti/fs_resources///credentials +~/.ti/fs_credentials///credentials ``` -Resource config files contain `file_system_name`, `tenant_id`, -`cloud_provider`, `region_code`, and `created_at`. Resource credentials files -contain only `api_key`, use mode `0600`, and must never be written to the main -`~/.ti/credentials` file. Profile and resource path segments are safely -encoded; always use the stored `file_system_name` for user-facing output. +Credential files contain `file_system_id`, canonical `region_code`, and +`api_key`, use mode `0600`, and must never be written to the main +`~/.ti/credentials` file. Profile and ID path segments are safely encoded. +Remote Drive9 list/get is authoritative for inventory and status; local state +only determines `has_local_token` and data-plane access. `ti fs create-file-system` returns the stored owner credential as `fs_token`; this is the only ordinary command result that may reveal it. Treat `fs_token` as a secret and never include it in logs, telemetry, debug output, errors, mount locators, non-secret config, or test diagnostics. -Legacy flat `fs_resource_name`, `fs_tenant_id`, `fs_cloud_provider`, -`fs_region_code`, and `fs_api_key` fields are migration input only. The first fs -command migrates a complete legacy resource into the registry and clears the -flat fields. Incomplete legacy state fails with -`fs.resource_credentials_incomplete`. +Legacy flat fields and name-keyed `~/.ti/fs_resources` entries are migration +input only. The first FS command copies complete credentials into the ID-keyed +store without deleting name-keyed source files or old companion homes. +Incomplete or conflicting state fails closed. DB SQL user credentials live outside the main credentials file: @@ -862,37 +866,37 @@ local profile namespace and must not cause ti to write local `[env]` sections. Generated ti fs state is always stored under the selected local profile: `--profile`, `TI_PROFILE`, or `default`. -ti fs resource selection order is: +ti fs data-plane resource selection order is: -1. Explicit `--file-system-name`. -2. `TI_FS_FILE_SYSTEM_NAME`. -3. Otherwise fail with `fs.missing_file_system_name` before credential loading, - endpoint resolution, companion startup, or a remote request. +1. Explicit `--file-system-id` or `TI_FS_FILE_SYSTEM_ID`. +2. If an explicit `--fs-token` or `TI_FS_TOKEN` exists, derive the ID from its + structured token claim and require any separately supplied ID to match. +3. Otherwise fail with `fs.missing_file_system_id` before endpoint resolution, + companion startup, or a remote request. -Never infer a ti fs resource from profile state, registry cardinality, -creation order, or deletion side effects. `TI_FS_FILE_SYSTEM_NAME` is an -explicit process-scoped selector and must not be persisted. +Never infer a ti fs resource from profile state, credential-store cardinality, +creation order, or deletion side effects. `TI_FS_FILE_SYSTEM_ID` is an +explicit process-scoped assertion and must not be persisted. Remote ti fs, fs-git, fs-journal, and owner fs-vault commands use this FS credential lookup order: 1. Explicit command-local `--fs-token`. 2. `TI_FS_TOKEN`. -3. The selected resource's `api_key` in its resource-scoped credentials file. +3. The selected ID's `api_key` in its ID-keyed credentials file. Those commands do not require TiDB Cloud public/private keys. A clean machine -can use an existing resource with a file-system name, canonical region, and FS -token supplied independently through flags or environment variables. Do not +can use an existing resource with only a canonical region and FS token; the ID +is derived in memory from the token. Do not persist ephemeral flag/environment credentials or create a synthetic `[env]` -profile. `ti fs create-file-system` and `ti fs delete-file-system` remain -TiDB Cloud-authenticated; deletion also requires the selected locally -registered resource and its owner token. +profile. `ti fs create-file-system`, remote list/describe, and +`ti fs delete-file-system` remain TiDB Cloud-authenticated. Delete requires an +ID but does not require a locally stored owner token. -The selector is available on ti fs data-plane/runtime commands and all -`fs-git`, `fs-journal`, and `fs-vault` subcommands. Creation, deletion, and -description require an explicit resource name where their command contract -declares it. Drain and unmount resolve an existing mount through its mount path -and locator instead of selecting a resource again. +The ID selector is available on ti fs data-plane/runtime commands and all +`fs-git`, `fs-journal`, and `fs-vault` subcommands. Creation accepts no ID; +description and deletion require an ID. Drain and unmount resolve an existing +mount through its mount path and locator instead of selecting a resource again. When implementing command handlers, detect whether `--profile` was explicitly set before calling `config.Load`; the root flag has a default value, but that @@ -954,8 +958,8 @@ create-db-sql-users` owns those credentials and must be idempotent: it creates or repairs the stable ti-managed read-only, read-write, and admin users for a cluster instead of creating a new group every time. -Generated `ti fs` resource API keys live only in the per-resource credentials -files under `~/.ti/fs_resources/`. User-facing docs and commands must call +Generated `ti fs` resource API keys live only in the ID-keyed credentials +files under `~/.ti/fs_credentials/`. User-facing docs and commands must call these `ti fs` API keys or resource credentials, never reference implementation API keys. Filesystem data-plane commands route through the installer-managed Drive9 companion binary named @@ -1001,11 +1005,11 @@ mount consumption path. companion records a drain control socket. WebDAV mounts flush through normal file close semantics and should not be expected to support drain. -When invoking the companion, resolve exactly one registry resource and build a +When invoking a data-plane companion command, resolve exactly one file system ID and build a sanitized environment: `HOME` from that resource's scoped companion directory, `DRIVE9_SERVER` from its resolved endpoint, `DRIVE9_REGION_CODE` from its canonical resource region, `DRIVE9_API_KEY` from its per-resource credentials, -and TiDB Cloud public/private keys only for provision/delete flows. Strip +and TiDB Cloud public/private keys only for remote inventory/create/describe/delete flows. Strip inherited `DRIVE9_*` values so user shell state cannot override ti selection. Debug and error output must redact TiDB Cloud keys, ti fs API keys, vault tokens, SQL credentials, file contents, and secret values. @@ -1106,8 +1110,8 @@ Current expectations: the focused `make live-e2e-` targets or the aggregate `make live-e2e`. They must use the `live-e2e` profile and verify the real API/command surface for every implemented spec. Implemented mutating commands - must have real live mutation coverage with resource names scoped to the test - run and cleanup that only targets resources created by that run. + must have real live mutation coverage with resource IDs captured from create + responses and cleanup that only targets resources created by that run. Do not require live cloud credentials for ordinary `go test ./...`. diff --git a/Makefile b/Makefile index 11cc02c..6ec5f9d 100644 --- a/Makefile +++ b/Makefile @@ -65,7 +65,7 @@ live-e2e-db: build $(LIVE_E2E_RUN) -run '^TestLiveDB' live-e2e-fs: build - $(LIVE_E2E_RUN) -run '^TestLive(FSResourceRegistryLifecycle|FSCommandSurface|FSConfigurationFreeAccess|FSDataPlaneLifecycle|FSMountRuntime|FSWebDAVMountRuntime)$$' + $(LIVE_E2E_RUN) -run '^TestLive(FSRemoteInventoryLifecycle|FSCommandSurface|FSConfigurationFreeAccess|FSDataPlaneLifecycle|FSMountRuntime|FSWebDAVMountRuntime)$$' live-e2e-fs-git: build $(LIVE_E2E_RUN) -run '^TestLiveFSGit' diff --git a/README.md b/README.md index e9dfe15..51cdb91 100644 --- a/README.md +++ b/README.md @@ -16,14 +16,14 @@ With `ti`, an agent can persist state between sessions, share files across sandb 1. Create a file system and obtain the file system token (performed once, outside the sandbox). ```shell -export TI_FS_TOKEN="$(ti fs create-file-system --file-system-name agent-workspace --region --wait --query fs_token --output text)" +export TI_FS_TOKEN="$(ti fs create-file-system --region --wait --query fs_token --output text)" ``` 2. Mount the filesystem to a local path and use it as a normal POSIX-compliant filesystem (performed within the sandbox) ```shell export TI_FS_TOKEN="" -ti fs mount-file-system --file-system-name agent-workspace --mount-path /path-to-workspace --region +ti fs mount-file-system --mount-path /path-to-workspace --region echo "Hello Sandbox Workspace!" >> /path-to-workspace/hello.txt ``` @@ -103,7 +103,7 @@ Automation should move to `TI_*` and `TIDB_CLOUD_*` environment variables. The v - Authentication: a TiDB Cloud Public Key and a Private Key from the [TiDB Cloud API Keys](https://tidbcloud.com/org-settings/api-keys) console. - Default region: one of aws-us-east-1, aws-us-west-2, aws-eu-central-1, aws-ap-northeast-1, aws-ap-southeast-1, or ali-ap-southeast-1. - - Regions support TiDB Cloud Filesystem: aws-us-east-1, aws-us-west-2, aws-ap-southeast-1, and ali-ap-southeast-1. + - Regions support TiDB Cloud Filesystem: aws-us-east-1, aws-us-west-2, aws-ap-southeast-1, or ali-ap-southeast-1. - Regions support TiDB Cloud Starter: aws-us-east-1, aws-us-west-2, aws-eu-central-1, aws-ap-northeast-1, aws-ap-southeast-1, or ali-ap-southeast-1. Set up a default profile with one command: @@ -156,37 +156,41 @@ An integration can add optional process-scoped attribution without changing a pr ### TiDB Cloud Filesystem +The following example uses `jq` to extract the server-assigned ID and one-time token from one create response. + ```shell mkdir ~/my-workspace -ti fs create-file-system --file-system-name my-workspace --wait -ti fs mount-file-system --file-system-name my-workspace --mount-path ~/my-workspace +umask 077 +ti fs create-file-system --wait > ./filesystem.json +export FILE_SYSTEM_ID="$(jq -r '.file_system_id' ./filesystem.json)" +export TI_FS_TOKEN="$(jq -r '.fs_token' ./filesystem.json)" +rm ./filesystem.json +ti fs mount-file-system --file-system-id "$FILE_SYSTEM_ID" --mount-path ~/my-workspace ``` Automatic mounting uses FUSE on Linux and WebDAV on macOS and Windows. macOS users can install macFUSE and explicitly add `--driver fuse` for the full FUSE experience. -One profile can manage multiple file systems. ti never infers which resource a command targets, so provide `--file-system-name` for one-off commands or set `TI_FS_FILE_SYSTEM_NAME` for repeated commands: +`ti fs list-file-systems` reads the region-scoped remote inventory through TiDB Cloud credentials. A profile can access multiple file systems, including resources created on another machine. Data-plane commands never infer a resource from the number of local credentials, so provide `--file-system-id` or set `TI_FS_FILE_SYSTEM_ID`: ```shell -ti fs create-file-system --file-system-name scratch ti fs list-file-systems -ti fs describe-file-system --file-system-name scratch -export TI_FS_FILE_SYSTEM_NAME=scratch +ti fs describe-file-system --file-system-id "$FILE_SYSTEM_ID" +export TI_FS_FILE_SYSTEM_ID="$FILE_SYSTEM_ID" ti fs list-files ``` -`create-file-system` returns an file system token (`fs_token`) in its JSON result. This is the file system owner credential and should be handled as a secret. A configured machine can provision a file system and capture the token without printing the full result: - -```shell -export TI_FS_TOKEN="$(ti fs create-file-system --file-system-name agent-workspace --wait --query fs_token --output text)" -``` +`create-file-system` does not accept a user-defined name. Drive9 assigns the stable `file_system_id`, and the command returns the owner credential as `fs_token` once in its JSON result. Treat it as a secret. The example above captures both fields from one provisioning request and removes the temporary owner-only JSON file immediately. An agent sandbox can then use that existing file system without running `ti configure` or providing TiDB Cloud API keys: ```shell export TI_FS_TOKEN="" -ti fs mount-file-system --file-system-name agent-workspace --mount-path /path_to_workspace --region aws-us-east-1 +export TI_REGION_CODE="aws-us-east-1" +ti fs mount-file-system --mount-path /path_to_workspace ``` +The token contains its file system ID, so a clean sandbox does not need `TI_FS_FILE_SYSTEM_ID`. Set that variable only as an optional consistency assertion. To persist an existing token on another configured or unconfigured machine, run `ti fs import-file-system-token --from-file ./fs-token`; subsequent commands can select its ID without resupplying the token. + ### TiDB Cloud Starter `ti db` currently implements TiDB Cloud Starter only. Commands that do not identify an existing cluster require `--db-cluster-type starter`; there is no implicit default. Commands with `--db-cluster-id` discover the authoritative service plan and route internally without accepting a type flag. Essential, Premium, Dedicated, unknown, and conflicting plan metadata are rejected before the requested product operation. @@ -228,6 +232,7 @@ ti db format-db-connection-string ti db execute-sql-statement ti fs create-file-system +ti fs import-file-system-token ti fs delete-file-system ti fs list-file-systems ti fs describe-file-system @@ -294,7 +299,7 @@ ti update ti update --target-version v0.1.1 ``` -`ti update` downloads and verifies both `ti` and its `ti-drive9` companion before replacing either binary in the user-writable install directory. It never requests sudo. The old `tdc update` command cannot migrate to a differently named executable; install `ti` once with the new installer instead. +`ti update` downloads and verifies both `ti` and its `ti-drive9` companion before replacing either binary in the user-writable install directory. It never requests sudo. The old `ti update` command cannot migrate to a differently named executable; install `ti` once with the new installer instead. ## Documentation diff --git a/docs/pingcap-docs/docs b/docs/pingcap-docs/docs index 2897f13..0a65ec5 160000 --- a/docs/pingcap-docs/docs +++ b/docs/pingcap-docs/docs @@ -1 +1 @@ -Subproject commit 2897f1300c435feb07f4f502f9e1ba6aec7f6f79 +Subproject commit 0a65ec54f3f150acb7ca4c39dee1a2d1f0394e4e diff --git a/docs/present.md b/docs/present.md index a39d190..f48de52 100644 --- a/docs/present.md +++ b/docs/present.md @@ -147,54 +147,53 @@ SQL 默认通过 HTTPS SQL API 执行，一次命令只执行一个 statement。 ## 4. 创建并管理 Filesystem -创建名为 `ti-demo-workspace` 的资源。一个 profile 可以注册多个 Filesystem；每个资源都必须通过名称显式选择： +创建一个由服务端分配稳定 ID 的资源。远端 inventory 是资源状态的权威来源，本地只保存按 ID 索引的访问凭证： ```bash -bin/ti fs create-file-system \ - --file-system-name ti-demo-workspace \ - --dry-run +bin/ti fs create-file-system --dry-run -bin/ti fs create-file-system \ - --file-system-name ti-demo-workspace \ - --wait +export FILE_SYSTEM_ID="$(bin/ti fs create-file-system \ + --wait \ + --query file_system_id \ + --output text)" bin/ti fs list-file-systems --output text bin/ti fs describe-file-system \ - --file-system-name ti-demo-workspace \ + --file-system-id "$FILE_SYSTEM_ID" \ --output text bin/ti fs check-file-system \ - --file-system-name ti-demo-workspace \ + --file-system-id "$FILE_SYSTEM_ID" \ --output text ``` -创建命令的 JSON 结果包含一次性的 `fs_token`。它是资源 owner credential，不能写入日志或公开传递。资源元数据和凭证分别存储在 `~/.ti/fs_resources///` 下，不写入主 `~/.ti/credentials`。 +创建命令的 JSON 结果包含一次性的 `fs_token`。它是资源 owner credential，不能写入日志或公开传递。凭证存储在 `~/.ti/fs_credentials///credentials`，不写入主 `~/.ti/credentials`。 ## 5. 使用 Data Plane 操作文件 ```bash bin/ti fs create-directory \ - --file-system-name ti-demo-workspace \ + --file-system-id "$FILE_SYSTEM_ID" \ --path /demo printf 'hello from data plane\n' | bin/ti fs copy-file \ - --file-system-name ti-demo-workspace \ + --file-system-id "$FILE_SYSTEM_ID" \ --from-stdin \ --to-remote /demo/from-data-plane.txt \ --tag source=data-plane \ --description "created through ti fs data plane" bin/ti fs list-files \ - --file-system-name ti-demo-workspace \ + --file-system-id "$FILE_SYSTEM_ID" \ --path /demo \ --output text bin/ti fs read-file \ - --file-system-name ti-demo-workspace \ + --file-system-id "$FILE_SYSTEM_ID" \ --path /demo/from-data-plane.txt bin/ti fs describe-file \ - --file-system-name ti-demo-workspace \ + --file-system-id "$FILE_SYSTEM_ID" \ --path /demo/from-data-plane.txt \ --output text ``` @@ -202,8 +201,8 @@ bin/ti fs describe-file \ Unix-style alias 只缩短命令名，flags 仍使用完整名称： ```bash -bin/ti fs ls --file-system-name ti-demo-workspace --path /demo --output text -bin/ti fs cat --file-system-name ti-demo-workspace --path /demo/from-data-plane.txt +bin/ti fs ls --file-system-id "$FILE_SYSTEM_ID" --path /demo --output text +bin/ti fs cat --file-system-id "$FILE_SYSTEM_ID" --path /demo/from-data-plane.txt ``` ## 6. 挂载并验证双向可见性 @@ -213,7 +212,7 @@ export MOUNT_PATH="/tmp/ti-demo-${DEMO_ID}" mkdir -p "$MOUNT_PATH" bin/ti fs mount-file-system \ - --file-system-name ti-demo-workspace \ + --file-system-id "$FILE_SYSTEM_ID" \ --mount-path "$MOUNT_PATH" ``` @@ -231,7 +230,7 @@ cat "$MOUNT_PATH/demo/from-data-plane.txt" printf 'hello from mount\n' > "$MOUNT_PATH/demo/from-mount.txt" bin/ti fs read-file \ - --file-system-name ti-demo-workspace \ + --file-system-id "$FILE_SYSTEM_ID" \ --path /demo/from-mount.txt ``` @@ -249,14 +248,14 @@ bin/ti fs drain-file-system --mount-path "$MOUNT_PATH" mkdir -p "$MOUNT_PATH/repos" bin/ti fs-git clone-git-workspace \ - --file-system-name ti-demo-workspace \ + --file-system-id "$FILE_SYSTEM_ID" \ --repo-url https://github.com/octocat/Hello-World.git \ --target-path "$MOUNT_PATH/repos/hello" \ --blobless \ --hydrate background bin/ti fs-git hydrate-git-workspace \ - --file-system-name ti-demo-workspace \ + --file-system-id "$FILE_SYSTEM_ID" \ --target-path "$MOUNT_PATH/repos/hello" git -C "$MOUNT_PATH/repos/hello" status --short @@ -266,7 +265,7 @@ git -C "$MOUNT_PATH/repos/hello" status --short ```bash bin/ti fs-git add-git-worktree \ - --file-system-name ti-demo-workspace \ + --file-system-id "$FILE_SYSTEM_ID" \ --base-path "$MOUNT_PATH/repos/hello" \ --worktree-path "$MOUNT_PATH/repos/hello-feature" \ --branch-name demo-feature @@ -274,7 +273,7 @@ bin/ti fs-git add-git-worktree \ git -C "$MOUNT_PATH/repos/hello-feature" status --short bin/ti fs-git remove-git-worktree \ - --file-system-name ti-demo-workspace \ + --file-system-id "$FILE_SYSTEM_ID" \ --worktree-path "$MOUNT_PATH/repos/hello-feature" \ --force ``` @@ -287,7 +286,7 @@ bin/ti fs-git remove-git-worktree \ export JOURNAL_ID="jrn-demo-${DEMO_ID}" bin/ti fs-journal create-journal \ - --file-system-name ti-demo-workspace \ + --file-system-id "$FILE_SYSTEM_ID" \ --journal-id "$JOURNAL_ID" \ --journal-kind agent \ --title "ti demo ${DEMO_ID}" \ @@ -295,23 +294,23 @@ bin/ti fs-journal create-journal \ --label demo=present bin/ti fs-journal append-journal-entries \ - --file-system-name ti-demo-workspace \ + --file-system-id "$FILE_SYSTEM_ID" \ --journal-id "$JOURNAL_ID" \ --entry-json '{"type":"demo.started"}' \ --entry-json '{"type":"demo.completed"}' bin/ti fs-journal read-journal-entries \ - --file-system-name ti-demo-workspace \ + --file-system-id "$FILE_SYSTEM_ID" \ --journal-id "$JOURNAL_ID" \ --output text bin/ti fs-journal search-journal-entries \ - --file-system-name ti-demo-workspace \ + --file-system-id "$FILE_SYSTEM_ID" \ --entry-type demo.completed \ --include-entries bin/ti fs-journal verify-journal \ - --file-system-name ti-demo-workspace \ + --file-system-id "$FILE_SYSTEM_ID" \ --journal-id "$JOURNAL_ID" \ --output text ``` @@ -324,13 +323,13 @@ Journal 是 append-only、可验证的 workflow ledger，不是普通文本日 printf 'demo-token\n' > /tmp/ti-demo-token.txt bin/ti fs-vault create-secret \ - --file-system-name ti-demo-workspace \ + --file-system-id "$FILE_SYSTEM_ID" \ --secret-name demo-service \ --field ENDPOINT=https://example.invalid \ --field API_TOKEN=@/tmp/ti-demo-token.txt bin/ti fs-vault read-secret \ - --file-system-name ti-demo-workspace \ + --file-system-id "$FILE_SYSTEM_ID" \ --secret-name demo-service \ --field ENDPOINT \ --format raw @@ -340,7 +339,7 @@ bin/ti fs-vault read-secret \ ```bash export TI_VAULT_TOKEN="$(bin/ti fs-vault create-grant \ - --file-system-name ti-demo-workspace \ + --file-system-id "$FILE_SYSTEM_ID" \ --agent-id demo-agent \ --scope demo-service/ENDPOINT \ --permission read \ @@ -348,14 +347,14 @@ export TI_VAULT_TOKEN="$(bin/ti fs-vault create-grant \ --token-only)" bin/ti fs-vault read-secret \ - --file-system-name ti-demo-workspace \ + --file-system-id "$FILE_SYSTEM_ID" \ --secret-name demo-service \ --field ENDPOINT \ --format raw \ --vault-token "$TI_VAULT_TOKEN" bin/ti fs-vault list-audit-events \ - --file-system-name ti-demo-workspace \ + --file-system-id "$FILE_SYSTEM_ID" \ --secret-name demo-service \ --limit 20 \ --output text @@ -377,17 +376,17 @@ rm -rf "$MOUNT_PATH" ```bash bin/ti fs-vault delete-secret \ - --file-system-name ti-demo-workspace \ + --file-system-id "$FILE_SYSTEM_ID" \ --secret-name demo-service rm -f /tmp/ti-demo-token.txt ``` -删除 Filesystem 资源及其本地 registry entry： +删除 Filesystem 资源及其本地 credential entry： ```bash bin/ti fs delete-file-system \ - --file-system-name ti-demo-workspace + --file-system-id "$FILE_SYSTEM_ID" ``` 删除演示 cluster： diff --git a/docs/priciples.md b/docs/priciples.md index fd2481e..8b78b52 100644 --- a/docs/priciples.md +++ b/docs/priciples.md @@ -136,20 +136,15 @@ tidb_cloud_public_key = "..." tidb_cloud_private_key = "..." ``` -The v0.2.x release line accepts the corresponding legacy `TDC_*` environment variables and legacy `tdc_public_key`/`tdc_private_key` credential fields. New names win only when both values are equal; conflicting old and new values fail before mutation. New writes use canonical names only. - -When only `~/.tdc` exists, the first non-update `ti` invocation atomically copies durable state into `~/.ti`, records an owner-only migration marker, and preserves the old directory. When both directories were created independently or the marker is invalid, `ti` fails instead of merging them. Active mounts started by the old executable must be drained and unmounted with `tdc` before migration. Every `ti update` form bypasses this migration and all local product state. - -One profile can own multiple Filesystem resources. Each resource has isolated metadata and credentials: +One profile can store credentials for multiple Filesystem resources. Drive9 remains authoritative for remote inventory; ti stores only the one-time token and its routing hint, keyed by the server-assigned file system ID: ```text -~/.ti/fs_resources///config -~/.ti/fs_resources///credentials +~/.ti/fs_credentials///credentials ``` -The resource config stores `file_system_name`, `tenant_id`, `cloud_provider`, `region_code`, and `created_at`. Its credentials file stores only the owner `api_key`. The main profile never stores a default resource name or resource API keys. +The credential file stores `file_system_id`, canonical `region_code`, and the owner `api_key`. The main profile never stores a default resource ID or resource API keys. -Legacy flat `fs_*` fields are migration input only. A complete legacy resource is migrated into the registry and the old fields are cleared. Incomplete legacy state fails explicitly. +Legacy flat `fs_*` fields and name-keyed `~/.ti/fs_resources` records are migration input only. Complete legacy records are copied into the ID-keyed credential store without deleting the source, preserving rollback safety. Incomplete or conflicting legacy state fails explicitly. DB SQL credentials are cluster-scoped because TiDB Cloud cluster IDs are globally unique: @@ -197,11 +192,11 @@ ti supplies a sanitized companion environment containing the resolved server, ca Filesystem resource selection is: -1. Explicit `--file-system-name`. -2. `TI_FS_FILE_SYSTEM_NAME`. -3. Otherwise fail with `fs.missing_file_system_name` before endpoint resolution, companion startup, or a remote call. +1. Explicit `--file-system-id` or `TI_FS_FILE_SYSTEM_ID`. +2. The file system ID embedded in an explicitly supplied FS token; any separate ID must match it. +3. Otherwise fail with `fs.missing_file_system_id` before endpoint resolution, companion startup, or a remote call. -ti never infers a target from profile state, the number of registered resources, creation order, or deletion side effects. Creating the first resource does not select it for later commands, and deleting a resource does not promote another resource. `TI_FS_FILE_SYSTEM_NAME` is an explicit process-scoped selector, not a persisted default. +ti never infers a target from profile state, the number of local credentials, creation order, or deletion side effects. Creating the first resource does not select it for later commands, and deleting a resource does not promote another resource. `TI_FS_FILE_SYSTEM_ID` is an explicit process-scoped selector, or a consistency assertion when a token is also supplied; it is not a persisted default. Remote data-plane, mount, Git, journal, and owner Vault commands select their FS token in this order: @@ -214,12 +209,11 @@ A clean agent sandbox can access an existing Filesystem using only: ```text TI_FS_TOKEN TI_REGION_CODE -TI_FS_FILE_SYSTEM_NAME ``` -These environment values form an in-memory command context and are not persisted. TiDB Cloud API keys remain required for `create-file-system` and `delete-file-system`; deletion also requires the resource to be registered locally. +These environment values form an in-memory command context and are not persisted. The ID is derived from the token. TiDB Cloud API keys remain required for remote FS create, list, describe, and delete; deletion requires an ID but no local owner token. -`create-file-system` returns `fs_token` once in its structured result. This owner credential must never appear in logs, telemetry, debug output, errors, non-secret config, or list/describe output. +Drive9 is authoritative for the region-scoped remote inventory. `create-file-system` accepts no user-defined name and returns the server-assigned `file_system_id` plus `fs_token` once in its structured result. This owner credential must never appear in logs, telemetry, debug output, errors, non-secret config, or list/describe output. A known token can be validated and persisted with `import-file-system-token`. On macOS and Windows, automatic mounting selects WebDAV. On Linux, automatic mounting selects FUSE. macOS users can install macFUSE and explicitly request `--driver fuse` for the full mount behavior. Vault mount requires FUSE and is unavailable on Windows. `drain-file-system` is meaningful only for a FUSE mount that exposes a drain control socket. diff --git a/docs/requirements/mount-file-system-config-free-mount.md b/docs/requirements/mount-file-system-config-free-mount.md index 8c3e154..3a78645 100644 --- a/docs/requirements/mount-file-system-config-free-mount.md +++ b/docs/requirements/mount-file-system-config-free-mount.md @@ -1,75 +1,63 @@ -# `ti fs mount-file-system` Configuration\-Free Mount +# `ti fs mount-file-system` Configuration-Free Mount ## Problem -Today `ti fs mount-file-system` requires an initialized `~/.ti/credentials` file, which in turn requires `ti configure` with TiDB Cloud API public/private keys\. This makes the mount command unusable in ephemeral environments \(CI/CD pipelines, E2B sandboxes, Docker containers\) where provisioning long\-lived API credentials is undesirable\. Users want a "mount and go" experience with only the information intrinsic to the filesystem itself\. +Ephemeral environments such as CI jobs, E2B sandboxes, and containers should not require `ti configure`, TiDB Cloud API keys, or a persistent `~/.ti` profile merely to mount an existing file system. ## Solution -Make `ti fs mount-file-system --file-system-name my-workspace --mount-path ~/my-workspace` work without any prior `ti configure` by accepting three pieces of information directly: +An FS token contains the Drive9 tenant ID that is the canonical `file_system_id`. A clean environment therefore needs only: -1. **file\-system\-name** — identifies the target File System +1. `--fs-token` or `TI_FS_TOKEN` for filesystem-scoped authentication and identity. +2. `--region` or `TI_REGION_CODE` for endpoint routing. +3. `--mount-path` for the local mount target. -2. **\-\-region\-code** flag or **TI\_REGION\_CODE** environment variable — specifies the deployment region - -3. **\-\-fs\-token** flag or **TI\_FS\_TOKEN** environment variable — the filesystem\-scoped authentication token - -With these three variables, the CLI has everything it needs to mount the filesystem — region for the API endpoint, FS identity, and a scoped credential — without a credentials file or API key pair\. +`--file-system-id` and `TI_FS_FILE_SYSTEM_ID` remain optional. When supplied with a token, they are consistency assertions and must match the tenant ID embedded in that token. ## CLI Interface ```bash -# Minimal mount — everything via flags +# Minimal sandbox mount using environment variables. +export TI_REGION_CODE=aws-us-east-1 +export TI_FS_TOKEN=drive9_abc123...xyz +ti fs mount-file-system --mount-path ~/my-workspace + +# Supply all authentication and routing values as flags. ti fs mount-file-system \ - --file-system-name my-workspace \ --mount-path ~/my-workspace \ --region aws-us-east-1 \ --fs-token drive9_abc123...xyz -# Mount using env vars for region and token -export TI_REGION_CODE=aws-us-east-1 -export TI_FS_TOKEN=drive9_abc123...xyz +# Optionally assert the expected file system ID. ti fs mount-file-system \ - --file-system-name my-workspace \ - --mount-path ~/my-workspace - -# Mount with mixed sources — pick what suits the environment -ti fs mount-file-system \ - --file-system-name my-workspace \ + --file-system-id tnt_abc123 \ --mount-path ~/my-workspace \ - --region aws-us-east-1 + --region aws-us-east-1 \ + --fs-token drive9_abc123...xyz ``` ## Variable Precedence -When multiple sources provide the same variable, the priority is \(highest wins\): - -1. CLI flag \(`--region`, `--fs-token`\) - -2. Environment variable \(`TI_REGION_CODE`, `TI_FS_TOKEN`\) +For each value independently, the priority is: -3. `~/.ti/credentials` \(existing config, if any\) +1. Explicit CLI flag. +2. Canonical environment variable. +3. Locally stored ID-keyed credential in the selected profile. +4. Profile region for endpoint routing. -This means existing configured setups continue to work unchanged — the new flags and env vars are purely additive\. +Canonical and legacy environment variables with different values fail explicitly. Values from different precedence layers can be combined, so a flag may provide the ID while the environment provides the token and the profile provides the region. ## Behavior -1. CLI resolves `region-code` and `fs-token` from flags → env vars → credentials file \(in order of precedence\) - -2. If either `region-code` or `fs-token` is missing after resolution, CLI prints a clear error telling the user which variable is missing and the available sources - -3. CLI connects to the Drive9 API endpoint using the region code and authenticates directly with the FS token - -4. Mount proceeds as normal — FUSE mount \(Linux/macOS\) or WebDAV \(fallback\) - -5. No `~/.ti/credentials` file is **pre\-required** before this mount command +1. Resolve the optional explicit ID, token, and region independently. +2. Parse the token and derive its tenant ID. +3. Reject an explicit ID that does not match the token. +4. Resolve the Drive9 endpoint from the canonical region code. +5. Mount through the bundled `ti-drive9` companion without requiring or creating profile files. ## Use Cases -- **E2B sandboxes:** pass `TI_FS_TOKEN` and `TI_REGION_CODE` as sandbox environment variables — no config step needed - -- **CI/CD pipelines:** inject token and region via CI secrets, mount in one command - -- **Docker containers:** pass via `--env` flags, no volume\-mounting credentials - -- **Quick trials:** a new user with only a token can mount immediately without API key setup \ No newline at end of file +- **E2B sandboxes:** inject `TI_FS_TOKEN` and `TI_REGION_CODE` and mount immediately. +- **CI/CD pipelines:** keep the token in CI secrets without distributing TiDB Cloud account keys. +- **Docker containers:** pass two environment variables without mounting the host credentials directory. +- **Configured machines:** select an ID-keyed local credential and omit repeated token input. diff --git a/docs/spec/0029-local-file-system-mount-inventory.md b/docs/spec/0029-local-file-system-mount-inventory.md new file mode 100644 index 0000000..cc608d4 --- /dev/null +++ b/docs/spec/0029-local-file-system-mount-inventory.md @@ -0,0 +1,277 @@ +# Local File System Mount Inventory + +## Goal + +Add a read-only command that reports where ordinary TiDB Cloud Filesystem resources are mounted on the current machine for the current operating-system user: + +```bash +ti fs list-local-file-system-mounts +ti fs list-local-file-system-mounts --file-system-id +``` + +The command must report both the local mount path and the remote path exposed through that mount. Its scope is deliberately local. It must not claim to list mounts created by another user, another `TI_HOME`, another machine, a container that does not share the same ti home, or an organization-wide backend inventory. + +## Product Decisions + +- The command name is `list-local-file-system-mounts`, not `list-file-system-mounts`, because ti has no server-side mount inventory API. +- The command lists ordinary `ti fs mount-file-system` mounts only. `ti fs-vault mount-vault` remains a separate security boundary and is not included. +- A file system may have multiple local mount paths, and every independently tracked mount is returned. +- `--file-system-id` is optional and filters the local result to one server-assigned file system ID. +- The command is read-only, does not accept `--dry-run`, and does not contact TiDB Cloud or Drive9. +- Listing local mounts does not require TiDB Cloud public/private keys, an FS token, or a network connection. +- The result is based on ti-owned locator state plus an operating-system mount check. A locator is not by itself proof that a mount is still active. +- The command never removes stale state. Cleanup remains an explicit unmount operation, including `ti fs unmount-file-system --mount-path --ignore-absent` when the mount is already gone. + +## User-facing Command + +List all ordinary ti filesystem mounts recorded under the current `TI_HOME`: + +```bash +ti fs list-local-file-system-mounts +``` + +Filter by file system ID: + +```bash +ti fs list-local-file-system-mounts --file-system-id tnt_abc123 +``` + +The global `--profile` flag follows a special local-inventory rule: when omitted, the command returns mounts from every profile represented in the current user's locator directory; when explicitly provided, it filters to that profile. This prevents the implicit `default` profile from hiding local mounts created through another profile while preserving an explicit profile filter for automation. + +The command supports the ordinary read-only output contract: + +```bash +ti fs list-local-file-system-mounts --output text +ti fs list-local-file-system-mounts --query 'mounts[?status == `mounted`].mount_path' +``` + +## Output Contract + +JSON output has one stable envelope: + +```json +{ + "mounts": [ + { + "file_system_id": "tnt_abc123", + "profile": "default", + "region_code": "aws-us-east-1", + "mount_path": "/home/user/workspace", + "remote_path": "/projects/demo", + "driver": "fuse", + "status": "mounted", + "foreground": false + } + ] +} +``` + +Fields: + +- `file_system_id`: the Drive9 tenant ID exposed by ti as the public filesystem identifier. +- `profile`: the ti profile namespace that created the isolated companion context. +- `region_code`: the canonical ti placement code. +- `mount_path`: the canonical absolute local path. +- `remote_path`: the normalized remote path mounted at `mount_path`. It is omitted only for a legacy locator that cannot be enriched safely. +- `driver`: the actual recorded driver, `fuse` or `webdav`. A legacy or in-progress record may report `unknown`; do not guess from the operating system. +- `status`: `mounted`, `stale`, or `unknown`. +- `foreground`: whether ti started the companion in foreground mode. + +An implementation may add an optional `pid` only when it comes from structured ti-owned process state. It must not scrape Drive9 human-readable stderr to obtain a PID, and callers must not depend on `pid` being present. + +Results are sorted first by `file_system_id`, then by canonical `mount_path`. An empty inventory succeeds with: + +```json +{"mounts": []} +``` + +Text output is a compact table containing at least file system ID, status, driver, local mount path, and remote path. It must not include API keys, FS tokens, companion home paths, state-file paths, log paths, or raw mount-helper output. + +## Mount Status Semantics + +Status is evaluated independently for each locator: + +- `mounted`: the platform mount inspection definitively reports that the canonical local path is an active mount point. +- `stale`: the locator is valid but platform inspection definitively reports that the path is not mounted. +- `unknown`: mount state cannot be determined safely because the platform does not expose a supported check, access is denied, or the locator represents a foreground mount that has not yet become observable. + +The command must not report `mounted` merely because a locator file exists or a PID appears alive. It must not treat a directory that merely exists as a mount. Platform checks must avoid cgo and use operating-system facilities or established dependencies already present in the repository: + +- Linux: inspect mount information such as `/proc/self/mountinfo` with correct path unescaping and exact mount-point matching. +- macOS: inspect the mounted-filesystem table through a supported Go/syscall path or a stable system command with bounded execution and structured parsing. +- Windows: use an available mount/volume check where it can distinguish an active mount; otherwise return `unknown` rather than a false positive. + +A failed inspection for one entry must not hide valid entries. The result may include non-secret warnings for unreadable or malformed locator files, but warnings must not expose file contents or companion credentials. + +## Locator Schema And Lifecycle + +Extend the ti-owned locator schema under `~/.ti/mounts/` from `ti.fs.mount-locator/v1` to a version that records the information needed for listing: + +```json +{ + "schema": "ti.fs.mount-locator/v2", + "profile": "default", + "file_system_id": "tnt_abc123", + "region_code": "aws-us-east-1", + "companion_home": "/home/user/.ti/drive9-home/...", + "mount_path": "/home/user/workspace", + "remote_path": "/projects/demo", + "driver": "fuse", + "foreground": false, + "kind": "fs" +} +``` + +`companion_home` remains internal routing state. It is required by drain and unmount but is never rendered by the list command. + +Background mount lifecycle: + +1. Resolve the file system ID, region, companion home, normalized remote path, and canonical local path. +2. Invoke the bundled `ti-drive9 mount` command. +3. After Drive9 confirms readiness and ti has determined the actual driver, atomically write the v2 locator. +4. If locator persistence fails, invoke the companion unmount command and return an error, preserving the existing all-or-nothing routing invariant. +5. A successful `drain-file-system` keeps the locator because the mount still exists. +6. A successful `unmount-file-system` removes the locator. + +Foreground mount lifecycle: + +1. Write a provisional v2 locator before entering the blocking companion command, after local validation succeeds. +2. Mark it as `foreground: true`; record an explicit requested driver or `unknown` when driver selection is automatic. +3. Keep the locator while the foreground process is running so another ti process can discover the mount. +4. Remove the locator when the foreground command exits, whether it exits normally, is cancelled, or fails startup. +5. Platform mount inspection, not the provisional record, determines whether the status is `mounted` or `unknown`. + +Locator writes and removal remain atomic and owner-only. Concurrent listing must see either the previous complete record or the new complete record, never a partially written JSON file. + +## Compatibility With Existing Locators + +Existing v1 locators must continue to support drain and unmount and must appear in the new list command where possible. The v1 field `file_system_name` is interpreted as the selected filesystem identity used by that release; after the remote-inventory migration, values that are valid Drive9 tenant IDs map to `file_system_id`. + +For a v1 locator: + +- Preserve profile, region, companion home, canonical mount path, and kind. +- Do not invent a remote path or driver that the locator did not record. +- Report missing recoverable fields as absent or `unknown`. +- Do not read or depend on private Drive9 source packages, fixtures, or undocumented on-disk process-state formats to enrich the result. +- Do not rewrite the locator merely because it was listed. The next successful mount at that path writes v2. +- If a legacy value cannot be represented as a valid file system ID, keep drain/unmount compatibility but return a non-secret warning instead of silently assigning it to another remote filesystem. + +This compatibility is a local-state schema migration only. It must not call remote list/get APIs and must not require that the remote filesystem still exists. + +## Implementation Design + +- `internal/fs/mountlocator` owns locator v1/v2 decoding, strict validation, atomic writes, canonical path identity, directory enumeration, deterministic sorting, and removal. +- `internal/fs` owns local inventory orchestration, optional file system/profile filtering, status evaluation, and public result models. +- Platform-specific files under `internal/fs` or a focused subpackage own mount-point inspection. Keep package names short and do not introduce cgo. +- `internal/cli` registers `ti fs list-local-file-system-mounts` as a read-only command and routes output through the existing JSON/text/query path. +- The existing mount, drain, and unmount handlers remain the only writers/removers of ordinary filesystem locators. +- Do not import any package from `ref/drive9` or make runtime/tests depend on `ref/`. + +The locator directory may contain malformed, unsupported, or unrelated files. Enumeration must consider only the expected `*.locator.json` files, reject symlinks and non-regular files, enforce bounded file sizes, and validate that the filename matches the hash of the canonical mount path before trusting a record. + +## API And Call Chain + +This command adds no TiDB Cloud or Drive9 backend API request. + +List flow: + +1. Resolve the current ti home without loading cloud credentials or migrating unrelated profile state. +2. Enumerate ti-owned ordinary filesystem locator files under `~/.ti/mounts/`. +3. Parse and validate supported locator schemas. +4. Apply an explicitly supplied profile filter and optional `--file-system-id` filter. +5. Inspect each local mount path through the platform-specific mount checker. +6. Sort the results deterministically. +7. Apply `--query`, then render JSON or text. + +Mount and unmount continue to call the public bundled Drive9 CLI. The list command must not invoke Drive9 merely to inspect local state. + +## Dependencies And Platform Impact + +- No new third-party dependency is expected. +- No cgo dependency is allowed. +- Linux and macOS must distinguish active and stale mounts. +- Windows must return honest `unknown` status where active mount detection cannot be implemented reliably. +- The command must work without network connectivity. +- The command must not require FUSE libraries merely to list WebDAV or stale mount records. + +## Tests + +Unit tests must cover: + +- v2 locator round-trip, modes, atomic replacement, and deterministic enumeration; +- backward-compatible v1 reads; +- multiple local paths for one file system; +- multiple file systems and profiles; +- omitted versus explicitly supplied `--profile` behavior; +- `--file-system-id` filtering and an empty match; +- stable sorting; +- canonical path and locator filename validation; +- malformed JSON, unsupported schema, oversized files, symlinks, and non-regular files; +- no token, API key, companion home, state path, or log path in rendered output or errors; +- platform checker results for mounted, stale, and unknown states; +- background mount writes v2 only after readiness; +- failed background mount leaves no locator; +- foreground mount creates a provisional locator and removes it on every exit path; +- drain retains the locator and unmount removes it; +- list never deletes a stale locator. + +Black-box `make e2e` coverage must use the fake companion to mount two local paths for one filesystem and one path for another filesystem, verify list/filter/query/text behavior, unmount one path, and verify only that path disappears. + +Focused `make live-e2e-fs` coverage must mount a temporary remote path, list local mounts, verify the returned file system ID plus exact local and remote paths, drain when supported, unmount, and verify the locator is absent. The test must clean up only its own mount and remote paths. + +## Documentation Updates + +When implemented: + +- Add the command to `README.md` and its folded all-commands inventory. +- Add one command-reference page with examples to the PingCAP Preview documentation. +- Explain that this is current-machine, current-user state and not a backend-wide answer to "where is this filesystem mounted?". +- Document stale status and explicit cleanup without advising users to delete locator files manually. + +## After This Spec + +A user can inspect all locally tracked mounts without remembering their mount paths: + +```bash +ti fs list-local-file-system-mounts --output text +``` + +An agent can locate active local paths for one filesystem: + +```bash +ti fs list-local-file-system-mounts \ + --file-system-id tnt_abc123 \ + --query 'mounts[?status == `mounted`].{local: mount_path, remote: remote_path}' +``` + +The result is sufficient for local drain/unmount orchestration but does not claim to discover mounts on other hosts. + +## Acceptance Criteria + +- `ti fs list-local-file-system-mounts` lists ordinary filesystem mounts recorded under the current user's ti home. +- Every new mount record includes the public file system ID, canonical local mount path, normalized remote path, region, profile, driver, and foreground mode. +- `--file-system-id` filters without requiring cloud credentials or a token. +- Omitted `--profile` lists all local profile namespaces; an explicitly supplied profile filters them. +- Active mounts are not inferred from locator existence alone. +- Stale records are clearly marked and are never removed by the list command. +- Existing v1 locators remain usable for drain/unmount and appear without fabricated fields. +- Foreground and background mount lifecycles cannot leave a locator after the owning mount command has definitively failed or exited. +- JSON, text, and `--query` behavior is deterministic and contains no secrets or internal companion paths. +- `make test`, `make e2e`, and focused live FS coverage pass. + +## Out Of Scope + +- Organization-wide or server-side mount inventory. +- Discovering mounts on another machine, container, user account, or ti home. +- A backend mount registration, heartbeat, lease, or last-seen API. +- Listing Vault mounts. +- Automatically pruning stale locator files. +- Terminating mount processes from the list command. +- Reading private Drive9 source packages or undocumented Drive9 state files. + +## Dependencies + +- `docs/spec/done/0015-drive9-companion-wrapper-for-tdc-fs.md` +- `docs/spec/done/0020-explicit-file-system-selection.md` +- `docs/spec/done/0027-ti-cli-rename-and-migration.md` +- `docs/spec/done/0028-remote-fs-resource-inventory.md` diff --git a/docs/spec/0030-remove-db-project-selection.md b/docs/spec/0030-remove-db-project-selection.md new file mode 100644 index 0000000..e2cd5d6 --- /dev/null +++ b/docs/spec/0030-remove-db-project-selection.md @@ -0,0 +1,379 @@ +# Remove DB Project Selection + +## Goal + +Remove client-side project selection from TiDB Cloud CLI configuration and every `ti db` workflow. TiDB Cloud Starter cluster creation must always omit project selection and let the TiDB Cloud service choose its server-side default project. + +Project is fading out as a user-facing TiDB Cloud concept. `ti` must not discover a default project, persist a project ID, accept a project selector, or infer a project from local state. + +This is an intentional breaking change that supersedes the active behavior originally introduced by `docs/spec/done/0017-default-virtual-project-resolution.md`. The completed spec remains unchanged as a historical record. + +## Product Decisions + +- Remove `--project-id` from `ti db create-db-cluster` without a compatibility alias or deprecation period. +- Do not add `TI_PROJECT_ID`, another environment variable, or another local project selector. +- `ti configure` does not discover `tidbx_virtual`, does not select a default project, and does not write `project_id`. +- `ti configure` becomes a local AWS CLI-style configuration operation. It validates local input and writes the selected profile without making a TiDB Cloud API request. +- Invalid or unauthorized API keys are reported by the first remote command that uses the permission required by that command, not by `ti configure`. +- A Starter create request omits project placement entirely. It must not send `project_id: ""`, `labels: {}`, or `labels: {"tidb.cloud/project": ""}`. +- TiDB Cloud remains free to return `labels["tidb.cloud/project"]` on cluster resources. ti preserves the API response and must not hide, rewrite, or interpret that server-selected label as local configuration. +- `ti organization list-projects` remains available as an independent organization inventory command. It is no longer part of configure or DB creation. +- Existing cluster, branch, IAM SQL-user, and SQL operations continue to identify resources through cluster and branch IDs. They must not acquire a project parameter. +- Future DB product providers must not reintroduce a generic `--project-id` on `ti db` without a new approved product design. + +## User-facing Behavior + +Configure a profile: + +```bash +ti configure +``` + +Non-interactive configuration remains: + +```bash +TI_REGION_CODE=aws-us-east-1 \ +TIDB_CLOUD_PUBLIC_KEY='' \ +TIDB_CLOUD_PRIVATE_KEY='' \ +ti configure --non-interactive +``` + +The successful JSON result becomes: + +```json +{ + "profile": "default", + "region_code": "aws-us-east-1", + "credentials_stored": true +} +``` + +The text result contains the profile, canonical default region code, and credential persistence status. It contains no project ID or project type. + +Create a Starter cluster: + +```bash +ti db create-db-cluster \ + --db-cluster-type starter \ + --db-cluster-name application-db \ + --wait +``` + +The following old invocation is invalid after this spec: + +```bash +ti db create-db-cluster \ + --db-cluster-type starter \ + --db-cluster-name application-db \ + --project-id project-123 +``` + +It fails as a normal unknown flag usage error with exit code `2`. ti must not silently ignore the supplied project ID because doing so would create the cluster in a different placement than the caller requested. + +## Configure Contract + +`ti configure` collects only: + +- profile namespace from `--profile`, `TI_PROFILE`, or `default`; +- canonical default region code; +- TiDB Cloud public API key; +- TiDB Cloud private API key. + +Configuration call chain: + +1. Resolve and validate the profile name. +2. Read the region and credentials from flags, canonical environment variables, or the interactive prompt according to the existing precedence rules. +3. Validate the canonical region syntax and required local values. +4. Atomically update the selected profile's non-secret config and secret credentials. +5. Remove a legacy `project_id` key from the selected profile if it exists. +6. Return the local configuration result. + +Configure must not: + +- resolve the IAM endpoint; +- call `GET /v1beta1/projects`; +- paginate projects; +- search for `type = "tidbx_virtual"`; +- require `organization.project.read`; +- call the Starter cluster API merely to probe credentials; +- fail because the machine is offline or a TiDB Cloud endpoint is temporarily unavailable. + +This deliberately changes the meaning of configure from "authenticate and discover a virtual project" to "persist local command inputs". A successful configure result means the local profile was written; it does not claim that the API keys were remotely authenticated. + +## Starter Create Request + +The public Starter API request remains: + +```text +POST /v1beta1/clusters +``` + +An illustrative request body is: + +```json +{ + "displayName": "application-db", + "region": { + "name": "regions/aws-us-east-1" + } +} +``` + +If a monthly spending limit is supplied, `spendingLimit` is added as before. The body must not contain project placement or an empty labels object. + +The service selects its default project. A successful response can include: + +```json +{ + "clusterId": "cluster-123", + "displayName": "application-db", + "servicePlan": "Starter", + "labels": { + "tidb.cloud/project": "server-selected-project-id" + } +} +``` + +ti returns those labels as received. The returned project label is resource metadata only; it is not copied into `~/.ti/config` and does not become a default for a later create request. + +## Other DB Operations + +No request other than Starter create currently sends a project selection. Preserve that boundary explicitly: + +- `list-db-clusters` calls the organization-level cluster list API and filters by effective region and verified product type. +- `describe-db-cluster`, `update-db-cluster`, and `delete-db-cluster` use the cluster ID. +- Branch commands use cluster ID and branch ID. +- SQL-user preparation and repair use cluster ID. +- Connection-string formatting and SQL execution use cluster ID plus locally managed SQL credentials. +- Product dispatch discovers the cluster service plan through cluster metadata and does not use a project ID. + +Do not add a project filter to list pagination, dispatch discovery, Starter guardrails, SQL credential paths, operation logs, or telemetry. + +## Dry-run Behavior + +`ti db create-db-cluster --dry-run` renders the same projectless request that normal execution sends: + +```json +{ + "request": { + "method": "POST", + "path": "/v1beta1/clusters", + "body": { + "displayName": "application-db", + "region": { + "name": "regions/aws-us-east-1" + } + } + } +} +``` + +The dry-run output must not contain `project_id`, `project-id`, `tidb.cloud/project`, or `labels`. A dry run may explain in a non-request description that TiDB Cloud selects the server-side default project, but that explanation must not be represented as a request field. + +## Local Config And Migration + +New profiles contain no project key: + +```toml +[default] +region_code = "aws-us-east-1" +``` + +Existing profiles can contain the legacy key: + +```toml +[default] +region_code = "aws-us-east-1" +project_id = "legacy-project-id" +``` + +Migration rules: + +- Every DB command ignores the legacy value immediately after upgrade. +- Loading a profile must not copy the legacy value into the runtime `config.Profile` used by DB services. +- Ordinary DB, organization, FS, update, and help commands do not rewrite the config merely to remove the value. +- The next successful `ti configure` for that profile removes its `project_id` while updating region and credentials. +- Reconfiguring one profile must not remove or change values in another profile. +- A legacy `project_id` with malformed or unexpected content must remain inert and must not block profile loading or a projectless DB request. +- No migration marker is required because the old field is non-secret, ignored, and safely removable on reconfigure. + +The config store may retain a private decode-only compatibility field for one release if needed to remove the old TOML key deterministically. It must not expose that field through the runtime profile or use it in any request. + +## Implementation Design + +### CLI + +In `internal/cli`: + +- remove the `project-id` flag registration from `create-db-cluster`; +- stop reading `project-id` in `createClusterOptions`; +- update help, usage, and command tests; +- keep `db-cluster-type`, `db-cluster-name`, spending limit, wait, and dry-run behavior unchanged. + +### Configure + +In `internal/config/configure`: + +- remove project-listing client construction, pagination, virtual-project selection, and related constants; +- remove `ProjectID` and `ProjectType` from the configure result; +- remove IAM resolver and transport requirements that exist only for project discovery; +- retain interrupt handling, secret input, region validation, environment precedence, profile selection, and owner-only credential persistence. + +In `internal/config/store` and `internal/config`: + +- stop exposing `ProjectID` on the runtime profile; +- provide a deterministic selected-profile write path that deletes legacy `project_id` during configure; +- preserve unrelated profile sections and credentials; +- do not turn an environment credential source into a persisted `[env]` profile. + +### DB Contracts And Starter Provider + +In `internal/db` and `internal/db/product/starter`: + +- remove `ProjectID` and `ProjectIDExplicit` from generic create options; +- delete default-project resolution logic and its errors, including `db.empty_project_id`; +- keep project concerns out of the generic provider contract. + +In `internal/api/starter`: + +- remove `ProjectID` from `CreateClusterRequest`; +- stop creating project labels in the request wire model; +- preserve response labels in the existing API model. + +No project-specific helper should remain in the create call path merely to pass an empty value. + +## Error Behavior + +After this spec: + +- `ti configure` can fail for invalid local input, interrupted input, unreadable state, or failed local writes. +- `ti configure` does not return IAM/project discovery errors such as `config.virtual_project_not_found`, `config.virtual_project_ambiguous`, or `config.invalid_virtual_project`. +- `ti db create-db-cluster --project-id ...` fails with Cobra's unknown-flag usage error. +- The first remote command reports authentication or authorization errors using that command's declared permission. +- A server-side create rejection is returned unchanged through the existing API error mapping. + +Do not add a warning merely because TiDB Cloud assigned a project label in the response. That is expected server behavior. + +## API Call Chain + +Configure: + +```text +flags/environment/prompts -> local validation -> ~/.ti/config + ~/.ti/credentials +``` + +There is no network call. + +Starter create: + +```text +ti db create-db-cluster + -> profile credentials and effective region + -> DB provider dispatch for starter + -> POST /v1beta1/clusters without project labels + -> TiDB Cloud selects project + -> ti validates Starter metadata and optionally waits for ACTIVE +``` + +Organization project listing remains explicit: + +```text +ti organization list-projects -> GET /v1beta1/projects +``` + +## Dependencies And Platform Impact + +- No new Go module is required. +- No cgo dependency is introduced. +- Configure becomes faster and works offline after the required local inputs are available. +- The change is identical on macOS, Linux, and Windows. +- This is a CLI and configure-output breaking change because `--project-id`, `project_id`, `project_type`, and the saved default project are removed. + +## Tests + +Unit tests must cover: + +- configure writes region and credentials without constructing or calling an IAM client; +- configure succeeds with an unreachable IAM/Starter endpoint because it performs no network request; +- configure JSON and text output contain no project fields; +- reconfiguring a profile removes only that profile's legacy `project_id`; +- a profile containing legacy `project_id` loads successfully but exposes no runtime project selection; +- create options and generic DB contracts contain no project input; +- Starter create request omits `labels` for every invocation, including profiles with a legacy project value; +- dry-run output omits project and labels fields; +- API response project labels remain present in rendered cluster output; +- `--project-id` is absent from help and rejected as an unknown flag; +- organization project listing remains functional and retains its own permission mapping. + +Black-box `make e2e` must cover: + +- interactive and non-interactive configure without an IAM test server; +- config and configure output with no project fields; +- a legacy config containing `project_id` followed by create dry-run and normal fake-API create, proving the request omits labels; +- explicit `--project-id` rejection; +- the unchanged `ti organization list-projects` command through its dedicated fake IAM server. + +`make live-e2e-configure` must verify local persistence without requiring project discovery. `make live-e2e-db` must create a real Starter cluster without an explicit or configured project ID, wait for `ACTIVE`, preserve the non-empty server-selected project label in the returned resource, and complete the existing branch, SQL, update, and delete lifecycle. The live profile loader must not run configure merely because `project_id` is absent. + +## Documentation Updates + +When implemented, update: + +- `docs/priciples.md` as the product source of truth; +- `AGENTS.md` current behavior, config examples, command examples, and live-e2e requirements; +- `README.md` configure and Starter creation workflows; +- current PingCAP Preview documentation for configure, credentials, Starter DB, create command reference, organization concepts, troubleshooting, and examples; +- release notes to identify removal of `--project-id` and configure result fields as a breaking change. + +Do not rewrite completed specs or archived release notes to pretend the previous default-project behavior never existed. + +## After This Spec + +Every Starter cluster created through ti uses TiDB Cloud's server-side default project selection: + +```bash +ti configure +ti db create-db-cluster \ + --db-cluster-type starter \ + --db-cluster-name application-db \ + --wait +``` + +Users who need project inventory can still request it explicitly: + +```bash +ti organization list-projects --output text +``` + +That inventory has no effect on later DB commands. + +## Acceptance Criteria + +- No public `ti db` command accepts a project ID. +- No `ti db` request sends a project ID or project-selection label. +- Starter create omits the project label rather than sending an empty value. +- TiDB Cloud can select the project and ti preserves the returned project label as resource metadata. +- `ti configure` performs no network request and stores no project ID. +- Existing profile `project_id` values are ignored immediately and removed only when that profile is reconfigured. +- Configure output contains no `project_id` or `project_type`. +- `ti organization list-projects` remains available but is not called implicitly. +- Dry-run, unit, black-box e2e, configure live-e2e, and DB live-e2e coverage prove the projectless request path. +- README and current product documentation match the implemented behavior. + +## Out Of Scope + +- Choosing or changing the TiDB Cloud service-side default project. +- Moving an existing cluster between projects. +- Hiding project labels returned by TiDB Cloud. +- Removing the explicit `ti organization list-projects` command. +- Modifying historical completed specs or old release notes. +- Designing project behavior for unimplemented Essential, Premium, or Dedicated providers. + +## Dependencies + +- `docs/spec/done/0002-local-config-and-credentials.md` +- `docs/spec/done/0005-organization-management.md` +- `docs/spec/done/0006-starter-db-cluster-lifecycle.md` +- `docs/spec/done/0017-default-virtual-project-resolution.md`, whose active behavior this spec supersedes +- `docs/spec/done/0026-db-provider-dispatch-and-starter-refactor.md` +- `docs/spec/done/0027-ti-cli-rename-and-migration.md` diff --git a/docs/spec/0028-homebrew-and-scoop-distribution.md b/docs/spec/0031-homebrew-and-scoop-distribution.md similarity index 100% rename from docs/spec/0028-homebrew-and-scoop-distribution.md rename to docs/spec/0031-homebrew-and-scoop-distribution.md diff --git a/docs/spec/0029-serverless-function-deployment.md b/docs/spec/0032-serverless-function-deployment.md similarity index 100% rename from docs/spec/0029-serverless-function-deployment.md rename to docs/spec/0032-serverless-function-deployment.md diff --git a/docs/spec/done/0009-tdc-fs-control-plane.md b/docs/spec/done/0009-tdc-fs-control-plane.md index f9da295..1bbc817 100644 --- a/docs/spec/done/0009-tdc-fs-control-plane.md +++ b/docs/spec/done/0009-tdc-fs-control-plane.md @@ -1,5 +1,7 @@ # tdc fs Control Plane +> **Latest identity update:** `0028-remote-fs-resource-inventory.md` supersedes the name-keyed local inventory. Current commands use server-assigned file system IDs and Drive9's region-scoped remote inventory. + > **Current status:** The original 1:1 profile model, flat `fs_*` storage, and native control-plane integration in this document are historical. `0015-drive9-companion-wrapper-for-tdc-fs.md` makes `tdc-drive9` the unconditional Filesystem implementation; `0016-profile-fs-resource-registry.md` provides profile-scoped 1:N resource storage; `0018-fs-token-auth-and-config-free-access.md` adds token-only use of existing resources; and `0020-explicit-file-system-selection.md` removes persistent default selection. The command intent and dry-run requirements below remain useful context. ## Goal diff --git a/docs/spec/done/0010-tdc-fs-data-plane.md b/docs/spec/done/0010-tdc-fs-data-plane.md index 6085bc5..86d914a 100644 --- a/docs/spec/done/0010-tdc-fs-data-plane.md +++ b/docs/spec/done/0010-tdc-fs-data-plane.md @@ -1,5 +1,7 @@ # tdc fs Data Plane +> **Latest identity update:** `0028-remote-fs-resource-inventory.md` supersedes file system name selectors. Current data-plane commands select a server-assigned ID or derive it from an explicitly supplied FS token. + > **Current status:** This document records the original command surface and tdc-native data-plane design. Since `0015-drive9-companion-wrapper-for-tdc-fs.md`, every retained public data-plane command is translated to the bundled `tdc-drive9` public CLI with no native fallback. Resource selection and credentials follow `0016-profile-fs-resource-registry.md` and `0018-fs-token-auth-and-config-free-access.md`; API keys are not stored in the main `~/.tdc/credentials`. Treat native HTTP, endpoint, upload, and filesystem-semantics statements below as historical. ## Goal diff --git a/docs/spec/done/0011-tdc-fs-mount-runtime.md b/docs/spec/done/0011-tdc-fs-mount-runtime.md index 189ade1..4b1d64b 100644 --- a/docs/spec/done/0011-tdc-fs-mount-runtime.md +++ b/docs/spec/done/0011-tdc-fs-mount-runtime.md @@ -1,5 +1,7 @@ # tdc fs Mount Runtime +> **Latest identity update:** `0028-remote-fs-resource-inventory.md` supersedes file system name selectors. Current mount commands select a server-assigned ID or derive it from an explicitly supplied FS token. + > **Current status:** This is the historical tdc-native mount design. `0015-drive9-companion-wrapper-for-tdc-fs.md` transferred FUSE, WebDAV, cache, write-back, drain, and unmount semantics to `tdc-drive9`; tdc now owns only command validation, resource/auth resolution, companion invocation, output/errors, and a non-secret background-mount locator. Automatic driver selection is FUSE on Linux and WebDAV on macOS and Windows; macOS users can install macFUSE and explicitly select FUSE. There is no native mount fallback. ## Goal diff --git a/docs/spec/done/0012-install-and-update-distribution.md b/docs/spec/done/0012-install-and-update-distribution.md index a06d857..e480889 100644 --- a/docs/spec/done/0012-install-and-update-distribution.md +++ b/docs/spec/done/0012-install-and-update-distribution.md @@ -2,7 +2,7 @@ ## Goal -Make `tdc` installable and updatable through deterministic GitHub Releases artifacts. The MVP channel is GoReleaser plus GitHub Releases, with shell and PowerShell installers. Homebrew and Scoop are intentionally deferred to `docs/spec/0028-homebrew-and-scoop-distribution.md`. +Make `tdc` installable and updatable through deterministic GitHub Releases artifacts. The MVP channel is GoReleaser plus GitHub Releases, with shell and PowerShell installers. Homebrew and Scoop are intentionally deferred to `docs/spec/0031-homebrew-and-scoop-distribution.md`. ## User-facing Commands @@ -266,7 +266,7 @@ Installer scripts: - Silent auto-update. - Updating TiDB Cloud credentials or DB SQL credentials. - Config migrations that modify user config during update. -- Homebrew tap and Scoop bucket publishing. See `0028-homebrew-and-scoop-distribution.md`. +- Homebrew tap and Scoop bucket publishing. See `0031-homebrew-and-scoop-distribution.md`. - Linux apt/yum repositories. - Winget publishing. - Notarization or binary signing beyond SHA-256 checksums for MVP. diff --git a/docs/spec/done/0014-tdc-fs-unix-command-aliases.md b/docs/spec/done/0014-tdc-fs-unix-command-aliases.md index 322ffe9..c137679 100644 --- a/docs/spec/done/0014-tdc-fs-unix-command-aliases.md +++ b/docs/spec/done/0014-tdc-fs-unix-command-aliases.md @@ -1,5 +1,7 @@ # tdc fs Unix Command Aliases +> **Latest identity update:** aliases use the ID/token selection contract in `0028-remote-fs-resource-inventory.md`; `--file-system-name` is no longer available. + ## Goal Add Unix-style aliases for common `tdc fs` file and mount operations while keeping the existing long, AWS-style command names as the canonical interface. The aliases make tdc easier for Linux and Drive9 users without changing flag names, output contracts, permissions, or implementation paths. diff --git a/docs/spec/done/0016-profile-fs-resource-registry.md b/docs/spec/done/0016-profile-fs-resource-registry.md index 2266e88..0f24263 100644 --- a/docs/spec/done/0016-profile-fs-resource-registry.md +++ b/docs/spec/done/0016-profile-fs-resource-registry.md @@ -1,8 +1,10 @@ # Profile FS Resource Registry +> **Latest identity update:** `0028-remote-fs-resource-inventory.md` supersedes this name-keyed inventory. The old registry is retained only as rollback-safe migration input; new local credentials are keyed by server-assigned ID. + This spec supersedes the 1:1 profile storage and flat `fs_*` credential rules in completed specs 0009 and 0015. -The persistent default-resource and unique-resource fallback rules in this completed spec are superseded by `docs/spec/done/0020-explicit-file-system-selection.md`. The 1:N registry and per-resource credential layout remain valid. +The persistent default-resource and unique-resource fallback rules in this completed spec are superseded by `docs/spec/done/0020-explicit-file-system-selection.md`. The 1:N product relationship remains valid, but the name-keyed inventory and credential layout are superseded by `0028-remote-fs-resource-inventory.md`. ## Goal diff --git a/docs/spec/done/0018-fs-token-auth-and-config-free-access.md b/docs/spec/done/0018-fs-token-auth-and-config-free-access.md index 41881b5..1fb596a 100644 --- a/docs/spec/done/0018-fs-token-auth-and-config-free-access.md +++ b/docs/spec/done/0018-fs-token-auth-and-config-free-access.md @@ -1,8 +1,10 @@ # FS Token Authentication And Configuration-Free Access +> **Latest identity update:** after `0028-remote-fs-resource-inventory.md`, a token-only sandbox needs only `TDC_FS_TOKEN` and `TDC_REGION_CODE`. The ID is derived from the token; `TDC_FS_FILE_SYSTEM_ID` is optional. + This spec refines `docs/requirements/mount-file-system-config-free-mount.md`. It keeps the configuration-free workflow but uses tdc's existing global `--region` contract, the profile-scoped FS resource registry introduced by `docs/spec/done/0016-profile-fs-resource-registry.md`, and the Drive9 companion ownership boundary from `docs/spec/done/0015-drive9-companion-wrapper-for-tdc-fs.md`. -The persistent default-resource and unique-resource fallback rules in this completed spec are superseded by `docs/spec/done/0020-explicit-file-system-selection.md`. Explicit `--file-system-name` and `TDC_FS_FILE_SYSTEM_NAME` selection, token precedence, and configuration-free access remain valid. +The persistent default-resource and unique-resource fallback rules in this completed spec are superseded by `docs/spec/done/0020-explicit-file-system-selection.md`. Token precedence and configuration-free access remain valid; the old name selectors are superseded by the ID/token selection contract in `0028-remote-fs-resource-inventory.md`. ## Goal diff --git a/docs/spec/done/0020-explicit-file-system-selection.md b/docs/spec/done/0020-explicit-file-system-selection.md index 273f9cc..bd12b8b 100644 --- a/docs/spec/done/0020-explicit-file-system-selection.md +++ b/docs/spec/done/0020-explicit-file-system-selection.md @@ -1,5 +1,7 @@ # Explicit File System Selection +> **Latest identity update:** `0028-remote-fs-resource-inventory.md` replaces explicit names with server-assigned IDs and permits token-derived ID selection. No default file system is inferred. + ## Goal Remove the persistent default-file-system experience from tdc. Every command that operates on a File System must receive its target explicitly from the current invocation. Data-plane and runtime commands resolve `--file-system-name` or `TDC_FS_FILE_SYSTEM_NAME`; control-plane commands that already require a resource name continue to require their explicit flag. tdc must not infer a target from profile state, registry cardinality, creation order, or deletion side effects. diff --git a/docs/spec/done/0028-remote-fs-resource-inventory.md b/docs/spec/done/0028-remote-fs-resource-inventory.md new file mode 100644 index 0000000..5980e5e --- /dev/null +++ b/docs/spec/done/0028-remote-fs-resource-inventory.md @@ -0,0 +1,404 @@ +# Remote File System Resource Inventory + +## Goal + +Make the Drive9 backend the source of truth for TiDB Cloud Filesystem resource inventory. Replace locally assigned file system names with one stable public identifier, `file_system_id`, whose value is the Drive9 tenant ID returned by provisioning. + +The local machine must no longer decide whether a file system exists. Local state is limited to the one-time owner token returned by `ti fs create-file-system`, the region routing hint required to use that token, and derived Drive9 companion runtime state. Losing local state must not prevent a user with valid TiDB Cloud API keys from listing, describing, or deleting remote file systems. + +This spec does not require Drive9 to return existing token plaintext or add token lifecycle APIs. Token generation, metadata listing, disable, enable, rotation, and revocation remain deferred until Drive9 exposes the required public API. + +## Product Decisions + +- A file system has no user-defined remote name. +- `file_system_id` is the public ti field and flag name. Its value is exactly the Drive9 tenant ID, for example `tnt_abc123`. +- Do not expose `tenant_id` as a second user-facing selector. Drive9 tenant terminology remains an implementation detail of TiDB Cloud Filesystem. +- Drive9 owns remote inventory, lifecycle status, and authorization binding. +- ti stores the owner token locally because it is returned only once and cannot be recovered through list or get APIs. +- `org:owner` and `project:owner` TiDB Cloud API keys are both accepted as organization-scoped operators, matching the current Drive9 authorization policy. +- Remote list, get, and delete are region-scoped. The effective ti region selects one Drive9 deployment. `--region` keeps its existing highest-priority override behavior. +- Do not add a server URL flag. Endpoint resolution continues to use the hosted Drive9 region manifest. + +## User-facing Commands + +The control-plane command surface becomes: + +```bash +ti fs create-file-system +ti fs create-file-system --wait +ti fs list-file-systems +ti fs describe-file-system --file-system-id +ti fs delete-file-system --file-system-id +ti fs import-file-system-token +``` + +`ti fs create-file-system` removes `--file-system-name`. The result includes the server-selected identifier and the one-time token: + +```json +{ + "file_system_id": "tnt_abc123", + "region_code": "aws-us-east-1", + "status": "provisioning", + "fs_token": "drive9_...", + "credentials_stored": true +} +``` + +All commands that select an existing file system replace `--file-system-name` with `--file-system-id`. This includes data-plane, mount, layer, `fs-git`, `fs-journal`, and `fs-vault` commands. The ID flag is optional only when an explicitly supplied FS token can identify the file system. Commands whose target is already identified by a mount path, such as drain and unmount, do not add a file system ID requirement. + +Configuration-free access becomes: + +```bash +TI_FS_TOKEN=drive9_... \ +TI_REGION_CODE=aws-us-east-1 \ +ti fs list-files --path / +``` + +The selector precedence for commands that require a file system is: + +1. `--file-system-id` +2. `TI_FS_FILE_SYSTEM_ID` +3. The `tenant_id` claim derived from an explicitly supplied `--fs-token` or `TI_FS_TOKEN`, after Drive9 accepts that exact token +4. No implicit default; fail with `fs.missing_file_system_id` + +When a flag or environment file system ID and an explicit token are both present, the supplied ID must match the verified token-derived ID. A mismatch fails before any data-plane operation. A token loaded from the local credential store cannot select its own record because an ID is required to locate that token in the 1:N store. + +`TI_FS_FILE_SYSTEM_NAME` and `--file-system-name` are removed after the migration behavior in this spec is implemented and tested. They must not silently select a different resource. + +## Import An Existing File System Token + +`ti fs import-file-system-token` imports a known Drive9 owner or filesystem-scoped token into the selected ti profile namespace. It is a local credential operation: it does not create a remote file system, issue a new token, change token status, or require TiDB Cloud public/private keys. + +The recommended invocation keeps the token out of command arguments: + +```bash +TI_FS_TOKEN='drive9_...' \ +ti fs import-file-system-token --region aws-us-east-1 +``` + +Automation may instead read the token from an owner-only file: + +```bash +ti fs import-file-system-token \ + --from-file ./fs-token \ + --region aws-us-east-1 +``` + +On POSIX systems, `--from-file` accepts only a regular file with mode `0600` or stricter. `--from-file -` reads one token from stdin. The command also accepts the existing `--fs-token` flag for consistency, but documentation must prefer `TI_FS_TOKEN`, stdin, or an owner-only file because command arguments can be exposed through shell history and process inspection. Supplying more than one token source is a usage error. + +The effective region follows the existing precedence of global `--region`, `TI_REGION_CODE`, and profile `region_code`. A region is required because Drive9 tokens are used only against the regional endpoint selected by ti. The command does not search other regions. + +A Drive9 API key looks opaque because `drive9_` is an outer wrapper, not the JWT itself. Its structure is: + +```text +drive9_ +``` + +After removing `drive9_` and Base64URL-decoding the remainder, the result is a standard signed JWT. Its payload includes `tenant_id`; ti maps that claim to the public `file_system_id`. Local decoding alone is not authentication because the payload is not yet signature-verified. + +Import therefore uses this validation chain: + +1. Parse the wrapper and JWT structure without logging or displaying any token bytes. +2. Extract the unverified `tenant_id` candidate and map it to `file_system_id`. +3. Run the bundled companion equivalent of `ti-drive9 fs stat --output json :/` with the exact token and selected regional endpoint in its sanitized environment. +4. Require Drive9 to accept the authenticated root metadata request. The Drive9 server verifies the signature, token version and status, and the token claim's binding to the authenticated tenant. ti does not depend on or reimplement the companion's underlying HTTP route. +5. Only after successful remote verification, atomically save the token under the derived `file_system_id`. + +The status response does not need to echo the tenant ID. A successful response proves that Drive9 accepted the exact wrapped JWT whose payload ti decoded. A malformed, expired, revoked, disabled, wrong-region, or otherwise rejected token is not written locally. + +The optional `--file-system-id ` is a caller assertion, not a second identity source. When provided, it must exactly equal the ID derived from the verified token or the command fails without writing. This is useful when an external system distributes the ID and token separately. + +Import is idempotent and fail-closed: + +- Importing the same ID, region, and token again succeeds without rewriting the credential. +- An existing entry for the same ID with a different token or region returns `fs.credential_import_conflict`. +- `--replace` explicitly permits replacement after the new token has passed remote verification. It never bypasses ID or region validation. +- `--dry-run` parses and remotely verifies the token and reports the derived ID and destination profile namespace without writing credentials or Drive9 context state. + +Successful structured output never returns the token: + +```json +{ + "file_system_id": "tnt_abc123", + "region_code": "aws-us-east-1", + "credentials_stored": true, + "status": "imported" +} +``` + +After import, commands still require explicit file system selection. They resolve the token from the ID-keyed credential store, so users no longer need to pass or export the token repeatedly: + +```bash +ti fs list-files --file-system-id tnt_abc123 --path / +ti fs mount-file-system --file-system-id tnt_abc123 --mount-path ./workspace +``` + +## Remote Inventory Behavior + +`ti fs list-file-systems` requires TiDB Cloud API keys from the selected profile or supported environment variables. It invokes the bundled Drive9 companion equivalent of: + +```bash +ti-drive9 admin tenant list \ + --region-code aws-us-east-1 \ + --tidbcloud-public-key \ + --tidbcloud-private-key \ + --json +``` + +The command follows Drive9 pagination until `next_page` is absent, uses the maximum supported page size, rejects repeated or regressing page values, and returns one aggregated deterministic result. Results are sorted by `file_system_id` so output does not depend on local directory order or backend page boundaries. + +Example JSON: + +```json +{ + "region_code": "aws-us-east-1", + "file_systems": [ + { + "file_system_id": "tnt_abc123", + "status": "active", + "kind": "live", + "has_local_token": true + }, + { + "file_system_id": "tnt_def456", + "status": "active", + "kind": "live", + "has_local_token": false + } + ] +} +``` + +`has_local_token` is a local capability hint, not remote resource state. It must never expose the token, token fingerprint, token length, local path, profile name, or Drive9 context name. + +A resource returned by Drive9 must appear even when the machine has no local token. A local credential absent from the remote result must not appear as a remote file system. The command may return a separate warning count for unmatched legacy credentials, but it must not merge stale local entries into `file_systems`. + +Token-only sandboxes cannot list the organization inventory because they do not have TiDB Cloud API keys. They can use only `TI_FS_TOKEN` and `TI_REGION_CODE` without a profile, local state, or separate file system ID. ti derives the ID from the token and Drive9 validates the binding on the first authenticated request. `TI_FS_FILE_SYSTEM_ID` remains an optional assertion when the sandbox receives the ID separately. + +## Describe And Delete Behavior + +`ti fs describe-file-system --file-system-id ` calls Drive9 tenant get with TiDB Cloud credentials. It does not require a local FS token. The output includes the canonical ti region, ID, status, kind, optional quota data returned by Drive9, and `has_local_token`. + +`ti fs delete-file-system --file-system-id ` calls Drive9 admin tenant delete with TiDB Cloud credentials. It must not require the owner token. This allows a user to delete a remote file system after changing machines or losing local ti state. + +Deletion remains asynchronous. After Drive9 accepts deletion, ti returns `status: "deleting"`. Only then may ti remove the new local credential entry for that ID and stop exposing it to new data-plane commands. Legacy registry files are not removed by this spec because they provide rollback safety. + +Delete dry-run validates the profile, TiDB Cloud credentials, region, endpoint, ID syntax, and required permission without calling Drive9 or deleting local credentials. It reports the Drive9 admin tenant delete operation and whether matching local credentials would be deactivated after acceptance. + +## Create And One-time Token Handling + +`ti fs create-file-system` invokes the bundled Drive9 public `create --json` provisioning command with the effective region and TiDB Cloud keys. ti does not send a name or spending limit. Drive9 returns the tenant ID, owner API key, status, provider, and region. Creation does not use the admin tenant API: `/v1/provision` remains the public provisioning contract, while the admin tenant API provides organization-scoped inventory, describe, and deletion. Because Drive9 `create` also writes a local context, ti runs it with an owner-only temporary companion Home, reads the structured response, and removes that Home. The returned token is persisted only in the ti ID-keyed credential store; create must not accumulate a second persistent inventory of Drive9 owner contexts. + +Before sending the create request, ti validates that the local credential directory is writable and that the Drive9 companion is available. After a successful response, ti atomically stores the owner token keyed by `file_system_id` and includes `fs_token` in the structured create result, preserving the existing ability to inject the token into a sandbox in one command. + +If the remote create succeeds but local persistence fails, ti must not hide the accepted resource or discard the one-time token. It returns a successful structured result with `credentials_stored: false`, includes the `file_system_id` and `fs_token`, writes an actionable warning to stderr in text mode, and does not retry creation. This exceptional success result is necessary because the token cannot currently be regenerated. + +Re-running create always requests a new remote resource. Local state must never make create return `exists`; idempotency cannot be inferred without a caller-supplied idempotency key or a server-side naming contract. + +`--wait` retains its current behavior: after creation, ti uses the returned token against the selected Drive9 endpoint until the root is readable or the timeout expires. Timeout and cancellation retain the remote resource and local token. + +## Local Credential Model + +New local credentials live under: + +```text +~/.ti/fs_credentials///credentials +``` + +The credential file is mode `0600` where POSIX permissions are available. Parent directories are mode `0700`. Directory keys remain encoded and path-safe even though current tenant IDs are safe strings. + +Example logical content: + +```toml +file_system_id = "tnt_abc123" +region_code = "aws-us-east-1" +api_key = "drive9_..." +``` + +`region_code` is an immutable routing hint captured with the token, not inventory state. Remote list/get remains authoritative for status and existence. An explicit global `--region` that conflicts with a locally stored routing hint fails before invoking Drive9; ti must not send a token to a different regional endpoint. + +The Drive9 companion may keep derived context and mount state under its isolated ti-owned home. That state is runtime material and must be reconstructible from file system ID, token, and region. It must not become a second resource inventory. + +## Migration From The Name-keyed Registry + +Existing installations store resources under: + +```text +~/.ti/fs_resources///config +~/.ti/fs_resources///credentials +``` + +The migration is automatic, lazy, idempotent, rollback-safe, and never creates or deletes a remote resource. It runs before the first FS command that loads local credentials in a profile. A process-scoped lock prevents concurrent migration within one process, and atomic file creation prevents partial destination records across processes. + +For every complete legacy resource: + +1. Read the old config and credentials without modifying them. +2. Validate the legacy file system name, tenant ID, canonical region, and non-empty API key using the existing strict parsers. +3. Treat the legacy tenant ID as the new `file_system_id`; the old file system name is migration metadata only and is not a remote identity. +4. Load a profile-scoped, non-secret migration completion list from the new credential directory. An ID already recorded as migrated is never recreated from the rollback source after its new credential is explicitly removed. +5. Preflight every not-yet-migrated legacy entry and every existing ID-keyed destination before writing any new destination. Migration must not call Drive9 or depend on TiDB Cloud keys, network availability, backend authorization rollout, or current remote existence. +6. Write each new ID-keyed credential file atomically with mode `0600`, then read it back and compare ID, region, and token before considering that entry migrated. +7. After all pending entries are stored, atomically update the owner-only migration completion list. A crash before this update is safe because matching credential writes are idempotent. +8. Reconstruct the new ID-keyed Drive9 companion context on demand. Do not move or delete an old companion home while a mount may still reference it. +9. Leave the original `~/.ti/fs_resources` entry untouched. A previous ti release can therefore still use it if the user rolls back. + +Migration establishes only a new local credential layout. It does not claim that a legacy resource still exists remotely. Subsequent remote list/get remains authoritative for existence and status, while a data-plane request remains authoritative for token validity. Keeping migration independent of remote inventory is required so an existing token remains usable while the admin inventory API is unavailable, denied, or temporarily failing. + +Conflict handling is fail-closed: + +- Existing destination with the same ID, region, and token is an idempotent success. +- Existing destination with a different token or region returns `fs.credential_migration_conflict`; do not choose one, overwrite either file, or call Drive9. +- Missing or malformed legacy config/credentials returns the existing incomplete-credential error and leaves all files unchanged. +- Two old names that map to the same tenant ID and same token collapse to one new credential entry and produce one non-fatal alias warning. +- Two old names that map to the same tenant ID with different tokens produce a conflict and retain every source file. + +The old name is not retained as a selector after successful migration. Users discover the stable ID through `ti fs list-file-systems`. Migration diagnostics may show a safe mapping from the old local name to the new file system ID, but must never include the token. + +This spec intentionally does not automatically delete legacy registry files. A later cleanup spec may remove them only after at least one release has used the new format, rollback support is no longer required, and active mount-state compatibility is proven. + +The migration completion list contains only schema version and file system IDs, never names, regions, tokens, token fingerprints, or paths. It is internal migration state rather than resource inventory. Remote deletion removes the active ID-keyed credential but retains this completion state, preventing a preserved rollback source from silently restoring access on the next command. + +## Resolution Rules After Migration + +For data-plane commands, ti resolves inputs in this order: + +1. Read an optional file system ID from `--file-system-id` or `TI_FS_FILE_SYSTEM_ID`. +2. Read an optional explicit token from `--fs-token` or `TI_FS_TOKEN`. +3. If no ID was supplied but an explicit token exists, decode its `tenant_id` claim as the candidate file system ID. If both exist, require them to match after remote token verification. +4. If an ID exists but no explicit token exists, load the matching token from the ID-keyed local credential file. Do not scan credential entries or select the only local entry implicitly. +5. Resolve the effective region from `--region`, `TI_REGION_CODE`, profile, or the matching local credential routing hint, while rejecting conflicts. Token-only use requires an explicit flag or environment region because there is no local routing hint. +6. Build or refresh the isolated Drive9 context from ID, token, and endpoint. +7. Invoke the bundled Drive9 public command. Drive9 remains responsible for cryptographically validating an explicitly supplied token before authorizing the requested operation. + +Inputs may be mixed across sources. For example, an explicit ID can use a token from the environment and region from the profile. No source creates an `[env]` profile or writes environment credentials to disk. + +## API And Companion Call Chain + +Remote list: + +1. Load profile TiDB Cloud public/private keys and effective region. +2. Resolve the hosted Drive9 endpoint from the bundled/hosted manifest. +3. Run `ti-drive9 admin tenant list --region-code --json` with credentials passed through the companion environment or explicit internal arguments without logging values. +4. Follow every Drive9 page and map `tenant_id` to `file_system_id`. +5. Join only `has_local_token` from the local ID-keyed credential store. +6. Apply JMESPath and render JSON or text through the existing shared output path. + +Remote describe and delete use `ti-drive9 admin tenant get/delete`. Create uses `ti-drive9 create --json` and the public `/v1/provision` endpoint. Data-plane commands continue to use the Drive9 owner or filesystem-scoped token interfaces. + +Local token import resolves the regional endpoint, validates the Drive9 wrapper and JWT payload using structured parsing, and invokes the bundled companion's public `fs stat` command against the remote root. ti must not import Drive9 Go packages to decode the token or reimplement the companion's HTTP request. The implementation may use the Go standard library for Base64URL and JSON parsing, while the Drive9 server remains the authority that cryptographically validates the token. + +Do not import Drive9 Go packages or code from `ref/`. ti integrates only through the bundled `ti-drive9` executable and its public command/output contract. + +## Authentication And Errors + +- List, describe, create, and delete require TiDB Cloud API keys and the existing ti FS control-plane permissions. +- Data-plane, mount, layer, Git, journal, and vault commands require an FS token but do not require TiDB Cloud API keys when ID and region are otherwise available. +- Drive9 `org:owner` and `project:owner` authorization is accepted without additional ti-side project filtering. +- A missing remote resource returns `fs.resource_not_found`, even if stale local credentials exist. +- A remotely visible resource without a local token is listable, describable, and deletable; data-plane use returns an actionable `auth.missing_fs_api_key` error and explains that token regeneration is not yet available. +- A user who possesses an existing token can run `ti fs import-file-system-token` to restore local data-plane access without recreating the file system or supplying TiDB Cloud API keys. +- A wrong-region ID returns the Drive9 not-found response mapped to `fs.resource_not_found`; ti must not search other regions implicitly. +- A token rejected during import returns an authentication error and leaves existing and destination credentials unchanged. +- API key values must never appear in logs, debug output, dry-run output, telemetry, or errors. + +## Package Design + +- `internal/fs`: remote control-plane orchestration and result mapping. +- `internal/fs/fscred`: new ID-keyed credential store, selector precedence, token import parsing and persistence, legacy migration, conflict handling, and `has_local_token` lookup. +- `internal/fswrap`: public Drive9 companion invocation for provisioning, admin tenant inventory, and data-plane commands. +- `internal/api/endpoints`: unchanged manifest-based region routing. +- `internal/cli`: flag migration from name to ID and shared output/dry-run wiring. +- `internal/config`: profile and TiDB Cloud credential loading only; do not put FS inventory into `~/.ti/config` or `~/.ti/credentials`. + +Keep one package per directory. Do not add a second filesystem inventory cache package or direct Drive9 HTTP client while the companion exposes the required public commands. + +## Dependencies And Platform + +- Requires a bundled Drive9 version that exposes public `create --json` plus `admin tenant list/get/delete` with stable JSON output in every supported ti FS region. +- Adds no Go runtime dependency and no cgo requirement. +- Retains the existing FUSE/WebDAV platform behavior because only control-plane discovery and credential selection change. +- Requires no Drive9 backend schema change for this phase because tenant ID is accepted as the sole resource identifier. +- Future token lifecycle support will require a separate Drive9 API and follow-up ti spec. + +The deployed Drive9 service is expected to enable `admin tenant list/get/delete` for ordinary TiDB Cloud organizations whose API keys have the accepted owner role, including organizations using free Starter capacity. The hosted region manifest determines which ti FS regions are currently available. Regional backend rollout is an external deployment concern and does not block completion of the ti client implementation. + +The server reference in `ref/fs/` confirms that ti uses the intended contract: `GET /v1/admin/tenants` with `X-TiDBCloud-Public-Key` and `X-TiDBCloud-Private-Key` headers, followed by organization-scoped tenant lookup. That reference revision rejects Free organizations in `authorizeTiDBCloudAdminAccess`, but the deployed service is newer and must be verified independently in each region. + +### Deployment Acceptance Status + +Verified on 2026-08-11 with the `live-e2e` TiDB Cloud credentials: + +| Region | Hosted manifest | Admin inventory | Lifecycle acceptance | +| --- | --- | --- | --- | +| `aws-us-east-1` | Published | List and describe pass | Create with `--wait`, list, describe, data-plane access, delete, and post-delete absence all pass for an isolated test resource. | +| `aws-ap-southeast-1` | Published | List and describe pass | Create with `--wait`, list, describe, data-plane access, delete, and post-delete absence all pass for an isolated test resource. | +| `aws-us-west-2` | Not published | A direct companion request to the known regional host returns `403 admin API is not available for free TiDB Cloud organizations`. | Not runnable until the deployment and manifest are updated. | +| `ali-ap-southeast-1` | Not published | No authoritative endpoint is available from the hosted manifest. | Not runnable until the deployment and manifest are updated. | + +The ti client implementation and offline acceptance suite are complete. The two regions published by the hosted manifest pass the isolated lifecycle. `aws-us-west-2` and `ali-ap-southeast-1` remain external Drive9 deployment work and require regional acceptance after publication. + +Five historical `aws-ap-southeast-1` tenants in the test organization remain visible through inventory but return a TiDB Cloud quota permission `403` when deleted with the current project-scoped API key. Newly created resources delete successfully through the same admin route. Cleaning those historical tenant bindings requires an API key with access to their underlying Starter resources or Drive9 backend intervention; it is not a ti client defect and does not reopen this spec. + +## Tests + +Unit tests must cover: + +- remote list pagination, deterministic sorting, empty results, malformed JSON, repeated page detection, and region routing; +- list output for resources with and without local tokens; +- describe/delete without an FS token and rejection without TiDB Cloud credentials; +- delete removes only the matching new credential after remote acceptance and preserves it after any failure; +- create stores the one-time token atomically and never returns `exists` from local state; +- create persistence failure returns the accepted ID and token once with `credentials_stored: false`; +- import derives the file system ID from a valid wrapped token, verifies it remotely, and stores it without TiDB Cloud credentials; +- import rejects malformed wrappers/JWTs, missing tenant claims, expired/revoked/disabled tokens, wrong regions, and caller-asserted ID mismatches without writing; +- import idempotency, conflict handling, explicit replacement, mutually exclusive token inputs, secure token-file permissions, stdin input, and dry-run behavior; +- selector precedence for `--file-system-id`, `TI_FS_FILE_SYSTEM_ID`, verified token-derived IDs, token sources, and region sources; +- token-only sandbox resolution with only `TI_FS_TOKEN` and `TI_REGION_CODE`, including rejection when an optional asserted ID does not match the verified token; +- no `--file-system-name` or `TI_FS_FILE_SYSTEM_NAME` dependency remains after migration; +- migration of one and multiple legacy resources; +- migration without TiDB Cloud keys; +- idempotent migration and every same-ID token/region conflict case; +- migration performs no companion or remote API call and remains available during remote authorization, not-found, and transient failures; +- no secret appears in output, errors, operation logs, telemetry, or dry-run results; +- old registry and old companion homes remain untouched. + +Black-box e2e must use a fake Drive9 companion to verify exact command arguments and mixed configuration sources. Live e2e must: + +1. Create a uniquely identified remote file system and retain its returned ID/token. +2. List it through remote inventory using TiDB Cloud keys. +3. Describe it by ID. +4. Use it through at least one data-plane command with locally stored credentials. +5. Copy the credential setup into a clean temporary ti home and prove list/describe work without the old registry. +6. Prove a known token and region work without TiDB Cloud credentials, a profile, local state, or `TI_FS_FILE_SYSTEM_ID`; repeat with an optional matching ID assertion and reject a mismatched assertion. +7. Remove its local credential, import the known token into the clean home, and prove subsequent data-plane use no longer requires a token flag or environment variable. +8. Delete by ID using TiDB Cloud credentials after removing the new local token from the clean home. +9. Confirm the resource disappears from remote list without deleting any pre-existing resource. + +Migration e2e fixtures must be created by the test itself under a temporary ti home. Tests must not depend on `ref/` fixtures or the developer's real `~/.ti` state. + +## Documentation Updates During Implementation + +Update README, AGENTS, completed FS specs with historical notes, installer next steps, and every English PingCAP ti FS command/reference/example page. Replace name-based examples and environment variables with file system IDs for locally stored credentials. Document that token-only sandboxes need only `TI_FS_TOKEN` and `TI_REGION_CODE`, while `TI_FS_FILE_SYSTEM_ID` is an optional consistency assertion. Explain that list is remote and region-scoped, while tokens remain local or explicitly injected. + +## Acceptance Criteria + +- A clean configured machine lists every authorized remote file system in its selected region without prior local FS registry state. +- A user can describe and delete a remote file system by ID after losing all local FS tokens. +- A user cannot read or mount a remote file system without a valid token. +- Existing name-keyed resources migrate to ID-keyed credentials without remote mutation, token loss, source deletion, or rollback breakage. +- A user can import a valid known token into an empty ti home and subsequently use that file system by ID without repeatedly supplying the token. +- Remote list results never include stale local-only resources. +- New creation does not accept or invent a file system name and returns the server-selected ID plus one-time token. +- Token-only sandboxes work with exactly token and region; a separately supplied ID is optional and must match the verified token. +- All regions currently published by the hosted manifest pass live create, list, describe, data-plane use, delete, and post-delete list verification. + +Future Drive9 regional rollouts require their own deployment acceptance checks. They do not change the completed ti command, credential, migration, or companion integration contracts in this spec. + +## Out Of Scope + +- User-defined remote file system names, aliases, or rename operations. +- Returning existing owner token plaintext from list/get. +- Token generation, rotation, disable, enable, metadata listing, or revocation. +- Automatic cross-region inventory aggregation or implicit region searching. +- Automatic deletion of legacy registry files. +- Changes to Drive9 backend authorization scope for `org:owner` or `project:owner`. diff --git a/docs/tdc-telemetry-metabase-dashboard.sql b/docs/tdc-telemetry-metabase-dashboard.sql new file mode 100644 index 0000000..f9f329e --- /dev/null +++ b/docs/tdc-telemetry-metabase-dashboard.sql @@ -0,0 +1,366 @@ +-- tdc telemetry dashboard queries for TiDB + Metabase. +-- +-- For scoped cards, configure these optional Metabase basic variables: +-- start_date -> Date +-- end_date -> Date +-- region_code -> Text +-- cli_version -> Text +-- +-- Leave every variable unset to query the complete global dataset. The global +-- KPI card intentionally has no variables and is never narrowed by dashboard filters. +-- +-- Use received_at, rather than occurred_at, as the dashboard time axis because +-- received_at is assigned by the backend and is not affected by client clock skew. +-- Success Rate and Wait Adoption are ratios from 0.0000 to 1.0000. Configure their +-- Metabase column type as Percentage; do not multiply them by 100 in SQL. +-- +-- METABASE SETUP +-- 1. Save each CARD below as a separate native SQL question. +-- 2. For cards 2-13, configure start_date and end_date as optional Date variables. +-- 3. Configure region_code and cli_version as optional Text variables. Prefer a +-- dropdown populated from telemetry_events.region_code or cli_version. +-- 4. Add four dashboard filters: From date, Through date, Region, and CLI version. +-- Connect them to start_date, end_date, region_code, and cli_version respectively. +-- 5. Do not connect dashboard filters to card 1. It is the all-time global baseline. +-- 6. Use UTC for the dashboard reporting timezone because received_at is backend time. +-- +-- RECOMMENDED DASHBOARD LAYOUT +-- Row 1: card 1 global KPI numbers and card 2 selected-scope KPI numbers. +-- Row 2: card 3 daily activity across the full width. +-- Row 3: card 4 command adoption, card 5 errors, and card 6 latency. +-- Row 4: card 7 Starter DB funnel and card 8 Filesystem funnel. +-- Row 5: card 9 repeat usage and card 10 version adoption. +-- Row 6: card 11 --wait adoption, card 12 platform distribution, and card 13 install +-- source distribution. + +-- CARD 1: Global lifetime KPIs +-- Visualization: four Number cards, one for each returned metric. In Metabase, +-- duplicate the question and retain one SELECT expression in each copy. A single-row +-- Table is an acceptable compact alternative. Do not connect dashboard filters. +SELECT + COUNT(DISTINCT anonymous_installation_id) AS `Active Installations`, + COUNT(*) AS `Command Invocations`, + COUNT(DISTINCT command_path) AS `Commands Used`, + ROUND(1.0 * SUM(CASE WHEN exit_code = 0 THEN 1 ELSE 0 END) / NULLIF(COUNT(*), 0), 4) AS `Success Rate` +FROM `tdc_telemetry`.`telemetry_events`; + +-- CARD 2: Scoped KPIs +-- Visualization: four Number cards, one for each returned metric. Connect all four +-- dashboard filters. With no variables set, this query also represents global data. +WITH scoped AS ( + SELECT * FROM `tdc_telemetry`.`telemetry_events` + WHERE 1 = 1 + [[AND received_at >= {{start_date}}]] + [[AND received_at < DATE_ADD({{end_date}}, INTERVAL 1 DAY)]] + [[AND region_code = {{region_code}}]] + [[AND cli_version = {{cli_version}}]] +) +SELECT + COUNT(DISTINCT anonymous_installation_id) AS `Active Installations`, + COUNT(*) AS `Command Invocations`, + COUNT(DISTINCT command_path) AS `Commands Used`, + ROUND(1.0 * SUM(CASE WHEN exit_code = 0 THEN 1 ELSE 0 END) / NULLIF(COUNT(*), 0), 4) AS `Success Rate` +FROM scoped; + +-- CARD 3: Daily activity +-- Visualization: Combo chart. Use Activity Date as the X-axis, bars for Command +-- Invocations, a line for Active Installations, and optionally a second-axis line for +-- Success Rate. Format Success Rate as Percentage in Metabase. +WITH scoped AS ( + SELECT * FROM `tdc_telemetry`.`telemetry_events` + WHERE 1 = 1 + [[AND received_at >= {{start_date}}]] + [[AND received_at < DATE_ADD({{end_date}}, INTERVAL 1 DAY)]] + [[AND region_code = {{region_code}}]] + [[AND cli_version = {{cli_version}}]] +) +SELECT + DATE(received_at) AS `Activity Date`, + COUNT(DISTINCT anonymous_installation_id) AS `Active Installations`, + COUNT(*) AS `Command Invocations`, + ROUND(1.0 * SUM(CASE WHEN exit_code = 0 THEN 1 ELSE 0 END) / NULLIF(COUNT(*), 0), 4) AS `Success Rate` +FROM scoped +GROUP BY DATE(received_at) +ORDER BY `Activity Date`; + +-- CARD 4: Command adoption and reliability +-- Visualization: Table sorted by Installations. Apply conditional formatting to +-- Success Rate and Failures, and format Success Rate as Percentage. For an +-- adoption-only view, use a horizontal bar chart with Command as the category and +-- Installations as the value. +WITH scoped AS ( + SELECT * FROM `tdc_telemetry`.`telemetry_events` + WHERE 1 = 1 + [[AND received_at >= {{start_date}}]] + [[AND received_at < DATE_ADD({{end_date}}, INTERVAL 1 DAY)]] + [[AND region_code = {{region_code}}]] + [[AND cli_version = {{cli_version}}]] +) +SELECT + command_path AS `Command`, + COUNT(DISTINCT anonymous_installation_id) AS `Installations`, + COUNT(*) AS `Invocations`, + SUM(CASE WHEN exit_code <> 0 THEN 1 ELSE 0 END) AS `Failures`, + ROUND(1.0 * SUM(CASE WHEN exit_code = 0 THEN 1 ELSE 0 END) / NULLIF(COUNT(*), 0), 4) AS `Success Rate`, + ROUND(AVG(duration_ms), 0) AS `Average Duration (ms)` +FROM scoped +GROUP BY command_path +ORDER BY `Installations` DESC, `Invocations` DESC; + +-- CARD 5: Top actionable errors +-- Visualization: Table with Failures and Affected Installations, or a horizontal +-- stacked bar chart using Command as the category, Failures as the value, and Error +-- Code as the series. +WITH scoped AS ( + SELECT * FROM `tdc_telemetry`.`telemetry_events` + WHERE 1 = 1 + [[AND received_at >= {{start_date}}]] + [[AND received_at < DATE_ADD({{end_date}}, INTERVAL 1 DAY)]] + [[AND region_code = {{region_code}}]] + [[AND cli_version = {{cli_version}}]] +) +SELECT + command_path AS `Command`, + COALESCE(NULLIF(error_code, ''), 'unclassified') AS `Error Code`, + COUNT(*) AS `Failures`, + COUNT(DISTINCT anonymous_installation_id) AS `Affected Installations` +FROM scoped +WHERE exit_code <> 0 +GROUP BY command_path, COALESCE(NULLIF(error_code, ''), 'unclassified') +ORDER BY `Failures` DESC, `Affected Installations` DESC +LIMIT 30; + +-- CARD 6: Successful-command latency, exact nearest-rank p50/p95 +-- Visualization: Table sorted by P95 Duration (ms). Apply conditional formatting to +-- that column and retain Samples so low-volume commands are not overinterpreted. +WITH scoped AS ( + SELECT * FROM `tdc_telemetry`.`telemetry_events` + WHERE exit_code = 0 + [[AND received_at >= {{start_date}}]] + [[AND received_at < DATE_ADD({{end_date}}, INTERVAL 1 DAY)]] + [[AND region_code = {{region_code}}]] + [[AND cli_version = {{cli_version}}]] +), ranked AS ( + SELECT + command_path, + duration_ms, + ROW_NUMBER() OVER (PARTITION BY command_path ORDER BY duration_ms) AS rank_no, + COUNT(*) OVER (PARTITION BY command_path) AS sample_count + FROM scoped +) +SELECT + command_path AS `Command`, + MAX(sample_count) AS `Samples`, + ROUND(AVG(duration_ms), 0) AS `Average Duration (ms)`, + MAX(CASE WHEN rank_no = CEIL(sample_count * 0.50) THEN duration_ms END) AS `P50 Duration (ms)`, + MAX(CASE WHEN rank_no = CEIL(sample_count * 0.95) THEN duration_ms END) AS `P95 Duration (ms)` +FROM ranked +GROUP BY command_path +HAVING MAX(sample_count) >= 5 +ORDER BY `P95 Duration (ms)` DESC; + +-- CARD 7: Starter DB activation funnel +-- Visualization: Funnel. Use Step as the stage and Installations as the value; sort +-- by Step Order ascending and hide Step Order from the displayed result when possible. +WITH scoped AS ( + SELECT * FROM `tdc_telemetry`.`telemetry_events` + WHERE 1 = 1 + [[AND received_at >= {{start_date}}]] + [[AND received_at < DATE_ADD({{end_date}}, INTERVAL 1 DAY)]] + [[AND region_code = {{region_code}}]] + [[AND cli_version = {{cli_version}}]] +), creators AS ( + SELECT anonymous_installation_id, MIN(received_at) AS created_at + FROM scoped + WHERE command_path = 'tdc db create-db-cluster' AND exit_code = 0 + GROUP BY anonymous_installation_id +), prepared AS ( + SELECT c.anonymous_installation_id, MIN(e.received_at) AS prepared_at + FROM creators c + JOIN scoped e ON e.anonymous_installation_id = c.anonymous_installation_id + AND e.command_path = 'tdc db create-db-sql-users' + AND e.exit_code = 0 + AND e.received_at >= c.created_at + GROUP BY c.anonymous_installation_id +), queried AS ( + SELECT p.anonymous_installation_id, MIN(e.received_at) AS queried_at + FROM prepared p + JOIN scoped e ON e.anonymous_installation_id = p.anonymous_installation_id + AND e.command_path = 'tdc db execute-sql-statement' + AND e.exit_code = 0 + AND e.received_at >= p.prepared_at + GROUP BY p.anonymous_installation_id +) +SELECT 1 AS `Step Order`, 'Created Starter cluster' AS `Step`, COUNT(*) AS `Installations` FROM creators +UNION ALL +SELECT 2, 'Created SQL users', COUNT(*) FROM prepared +UNION ALL +SELECT 3, 'Executed SQL', COUNT(*) FROM queried +ORDER BY `Step Order`; + +-- CARD 8: Filesystem activation funnel +-- Visualization: Funnel. Use Step as the stage and Installations as the value; sort +-- by Step Order ascending and hide Step Order from the displayed result when possible. +WITH scoped AS ( + SELECT * FROM `tdc_telemetry`.`telemetry_events` + WHERE 1 = 1 + [[AND received_at >= {{start_date}}]] + [[AND received_at < DATE_ADD({{end_date}}, INTERVAL 1 DAY)]] + [[AND region_code = {{region_code}}]] + [[AND cli_version = {{cli_version}}]] +), creators AS ( + SELECT anonymous_installation_id, MIN(received_at) AS created_at + FROM scoped + WHERE command_path = 'tdc fs create-file-system' AND exit_code = 0 + GROUP BY anonymous_installation_id +), accesses AS ( + SELECT + c.anonymous_installation_id, + COUNT(*) AS access_count + FROM creators c + JOIN scoped e ON e.anonymous_installation_id = c.anonymous_installation_id + AND e.command_path IN ( + 'tdc fs mount-file-system', + 'tdc fs copy-file', + 'tdc fs read-file', + 'tdc fs list-files' + ) + AND e.exit_code = 0 + AND e.received_at >= c.created_at + GROUP BY c.anonymous_installation_id +) +SELECT 1 AS `Step Order`, 'Created filesystem' AS `Step`, COUNT(*) AS `Installations` FROM creators +UNION ALL +SELECT 2, 'Accessed filesystem', COUNT(*) FROM accesses +UNION ALL +SELECT 3, 'Repeated filesystem access', COALESCE(SUM(CASE WHEN access_count >= 2 THEN 1 ELSE 0 END), 0) FROM accesses +ORDER BY `Step Order`; + +-- CARD 9: Repeat usage within the selected period +-- Visualization: Vertical bar chart. Use Active Day Bucket as the X-axis and +-- Installations as the Y-axis. Preserve the SQL result order. +WITH scoped AS ( + SELECT * FROM `tdc_telemetry`.`telemetry_events` + WHERE 1 = 1 + [[AND received_at >= {{start_date}}]] + [[AND received_at < DATE_ADD({{end_date}}, INTERVAL 1 DAY)]] + [[AND region_code = {{region_code}}]] + [[AND cli_version = {{cli_version}}]] +), installation_activity AS ( + SELECT + anonymous_installation_id, + COUNT(DISTINCT DATE(received_at)) AS active_days + FROM scoped + GROUP BY anonymous_installation_id +), buckets AS ( + SELECT + CASE + WHEN active_days = 1 THEN '1 day' + WHEN active_days BETWEEN 2 AND 3 THEN '2-3 days' + WHEN active_days BETWEEN 4 AND 7 THEN '4-7 days' + ELSE '8+ days' + END AS active_day_bucket, + CASE + WHEN active_days = 1 THEN 1 + WHEN active_days BETWEEN 2 AND 3 THEN 2 + WHEN active_days BETWEEN 4 AND 7 THEN 3 + ELSE 4 + END AS bucket_order + FROM installation_activity +) +SELECT active_day_bucket AS `Active Day Bucket`, COUNT(*) AS `Installations` +FROM buckets +GROUP BY active_day_bucket, bucket_order +ORDER BY bucket_order; + +-- CARD 10: CLI version adoption +-- Visualization: Horizontal bar chart with CLI Version as the category and +-- Installations as the value. Use the Table visualization when Success Rate also needs +-- to be compared, and format Success Rate as Percentage. +WITH scoped AS ( + SELECT * FROM `tdc_telemetry`.`telemetry_events` + WHERE 1 = 1 + [[AND received_at >= {{start_date}}]] + [[AND received_at < DATE_ADD({{end_date}}, INTERVAL 1 DAY)]] + [[AND region_code = {{region_code}}]] + [[AND cli_version = {{cli_version}}]] +) +SELECT + cli_version AS `CLI Version`, + COUNT(DISTINCT anonymous_installation_id) AS `Installations`, + COUNT(*) AS `Invocations`, + ROUND(1.0 * SUM(CASE WHEN exit_code = 0 THEN 1 ELSE 0 END) / NULLIF(COUNT(*), 0), 4) AS `Success Rate` +FROM scoped +GROUP BY cli_version +ORDER BY `Installations` DESC, `Invocations` DESC; + +-- CARD 11: --wait adoption on supported create/delete commands +-- Visualization: Horizontal bar chart with Command as the category and Wait Adoption +-- as the value. Format Wait Adoption as Percentage. Show Invocations in the tooltip or +-- use a Table when sample size needs to remain visible. +WITH scoped AS ( + SELECT * FROM `tdc_telemetry`.`telemetry_events` + WHERE command_path IN ( + 'tdc db create-db-cluster', + 'tdc db create-db-cluster-branch', + 'tdc db delete-db-cluster', + 'tdc fs create-file-system' + ) + [[AND received_at >= {{start_date}}]] + [[AND received_at < DATE_ADD({{end_date}}, INTERVAL 1 DAY)]] + [[AND region_code = {{region_code}}]] + [[AND cli_version = {{cli_version}}]] +) +SELECT + command_path AS `Command`, + COUNT(*) AS `Invocations`, + SUM(CASE WHEN JSON_CONTAINS(flag_names_json, JSON_QUOTE('wait')) = 1 THEN 1 ELSE 0 END) AS `Wait Invocations`, + ROUND( + 1.0 * SUM(CASE WHEN JSON_CONTAINS(flag_names_json, JSON_QUOTE('wait')) = 1 THEN 1 ELSE 0 END) + / NULLIF(COUNT(*), 0), + 4 + ) AS `Wait Adoption` +FROM scoped +GROUP BY command_path +ORDER BY `Invocations` DESC; + +-- CARD 12: Platform distribution by operating system and architecture +-- Visualization: Row chart. Click Row, then the gear icon, then Display, and select +-- Stack. Use Operating System as the category, Architecture as the series, and +-- Installations as the value. Do not use Stack - 100% because this card compares +-- absolute adoption volume. +WITH scoped AS ( + SELECT * FROM `tdc_telemetry`.`telemetry_events` + WHERE 1 = 1 + [[AND received_at >= {{start_date}}]] + [[AND received_at < DATE_ADD({{end_date}}, INTERVAL 1 DAY)]] + [[AND region_code = {{region_code}}]] + [[AND cli_version = {{cli_version}}]] +) +SELECT + os AS `Operating System`, + arch AS `Architecture`, + COUNT(DISTINCT anonymous_installation_id) AS `Installations` +FROM scoped +GROUP BY os, arch +ORDER BY `Operating System`, `Installations` DESC; + +-- CARD 13: Installation-source distribution +-- Visualization: Row chart. Use Install Source as the category and Installations as +-- the value. This is intentionally separate from card 12: a stacked Row chart can +-- clearly represent one category plus one series, not OS, architecture, and install +-- source at once. +WITH scoped AS ( + SELECT * FROM `tdc_telemetry`.`telemetry_events` + WHERE 1 = 1 + [[AND received_at >= {{start_date}}]] + [[AND received_at < DATE_ADD({{end_date}}, INTERVAL 1 DAY)]] + [[AND region_code = {{region_code}}]] + [[AND cli_version = {{cli_version}}]] +) +SELECT + COALESCE(NULLIF(install_source, ''), 'unknown') AS `Install Source`, + COUNT(DISTINCT anonymous_installation_id) AS `Installations` +FROM scoped +GROUP BY COALESCE(NULLIF(install_source, ''), 'unknown') +ORDER BY `Installations` DESC; diff --git a/e2e/cli_test.go b/e2e/cli_test.go index dc135df..1d27804 100644 --- a/e2e/cli_test.go +++ b/e2e/cli_test.go @@ -2,6 +2,7 @@ package e2e import ( "bytes" + "encoding/base64" "encoding/json" "fmt" "io" @@ -72,7 +73,8 @@ func TestHelpAndVersion(t *testing.T) { deleteFileSystem := runTI(t, bin, "fs", "delete-file-system", "help") deleteFileSystem.wantExitCode(0) - deleteFileSystem.wantStdoutContains("--file-system-name") + deleteFileSystem.wantStdoutContains("--file-system-id") + deleteFileSystem.wantStdoutNotContains("--file-system-name") deleteFileSystem.wantStdoutNotContains("--confirm-file-system-name") createDBCluster := runTI(t, bin, "db", "create-db-cluster", "help") @@ -152,7 +154,7 @@ func TestErrorsAreRenderedAtCLIBoundary(t *testing.T) { unknown.wantExitCode(2) unknown.wantStderrContains(`ti [ERROR]: unknown command "missing-command" for "ti db"`) - removedConfirmation := runTI(t, bin, "fs", "delete-file-system", "--file-system-name", "workspace", "--confirm-file-system-name", "workspace") + removedConfirmation := runTI(t, bin, "fs", "delete-file-system", "--file-system-id", "tenant-workspace", "--confirm-file-system-name", "workspace") removedConfirmation.wantExitCode(2) removedConfirmation.wantStderrContains(`unknown flag: --confirm-file-system-name`) @@ -619,7 +621,7 @@ func TestConfigureNonInteractiveFromEnvironment(t *testing.T) { } } -func TestFSResourceRegistrySelectionAcrossCommandFamilies(t *testing.T) { +func TestFSRemoteInventoryAndIDCredentialSelectionAcrossCommandFamilies(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("fake companion build path is covered by unit tests on Windows") } @@ -632,6 +634,7 @@ func TestFSResourceRegistrySelectionAcrossCommandFamilies(t *testing.T) { t.Fatalf("build fake Drive9 companion: %v\n%s", err, output) } recordPath := filepath.Join(t.TempDir(), "calls.jsonl") + statePath := filepath.Join(t.TempDir(), "state.json") manifestServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = fmt.Fprint(w, `{"service":"drive9","regions":[{"region_code":"aws-us-east-1","mode":"tidb_cloud_native","server_url":"https://fs-east.test","cloud_provider":"aws","tidb_region":"us-east-1"},{"region_code":"aws-us-west-2","mode":"tidb_cloud_native","server_url":"https://fs-west.test","cloud_provider":"aws","tidb_region":"us-west-2"}]}`) })) @@ -640,6 +643,7 @@ func TestFSResourceRegistrySelectionAcrossCommandFamilies(t *testing.T) { "HOME=" + home, "TI_DRIVE9_BIN=" + companion, "FAKE_DRIVE9_RECORD=" + recordPath, + "FAKE_DRIVE9_STATE=" + statePath, "TI_ALLOW_TEST_ENDPOINTS=1", "TI_TEST_FS_MANIFEST_URL=" + manifestServer.URL, } @@ -652,34 +656,38 @@ func TestFSResourceRegistrySelectionAcrossCommandFamilies(t *testing.T) { configured.wantExitCode(0) missingWithZeroResources := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "fs", "list-files", "--path", "/") missingWithZeroResources.wantExitCode(2) - missingWithZeroResources.wantStderrContains("file system name is required; pass --file-system-name or set TI_FS_FILE_SYSTEM_NAME") + missingWithZeroResources.wantStderrContains("file system ID is required") - createWorkspace := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "fs", "create-file-system", "--file-system-name", "workspace", "--wait") + createWorkspace := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "fs", "create-file-system", "--wait") createWorkspace.wantExitCode(0) createWorkspace.wantStdoutContains(`"status": "ready"`) createWorkspace.wantStdoutContains(`"credentials_stored": true`) - createWorkspace.wantStdoutContains(`"fs_token": "key-workspace"`) + createWorkspace.wantStdoutContains(`"file_system_id": "tenant-aws-us-east-1"`) missingWithOneResource := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "fs", "list-files", "--path", "/") missingWithOneResource.wantExitCode(2) - missingWithOneResource.wantStderrContains("file system name is required; pass --file-system-name or set TI_FS_FILE_SYSTEM_NAME") - createScratch := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "--region", "aws-us-west-2", "fs", "create-file-system", "--file-system-name", "scratch", "--wait") + missingWithOneResource.wantStderrContains("file system ID is required") + createScratch := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "--region", "aws-us-west-2", "fs", "create-file-system", "--wait") createScratch.wantExitCode(0) createScratch.wantStdoutContains(`"status": "ready"`) createScratch.wantStdoutContains(`"credentials_stored": true`) list := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "fs", "list-file-systems") list.wantExitCode(0) - list.wantStdoutContains(`"file_system_name": "workspace"`) - list.wantStdoutContains(`"file_system_name": "scratch"`) - list.wantStdoutNotContains("key-workspace") - list.wantStdoutNotContains("key-scratch") + list.wantStdoutContains(`"file_system_id": "tenant-aws-us-east-1"`) + list.wantStdoutNotContains(`"file_system_id": "tenant-aws-us-west-2"`) + list.wantStdoutContains(`"has_local_token": true`) + list.wantStdoutNotContains("drive9_") list.wantStdoutNotContains("default_file_system_name") list.wantStdoutNotContains("is_default") - describe := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "fs", "describe-file-system", "--file-system-name", "scratch") + westList := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "--region", "aws-us-west-2", "fs", "list-file-systems") + westList.wantExitCode(0) + westList.wantStdoutContains(`"file_system_id": "tenant-aws-us-west-2"`) + westList.wantStdoutNotContains(`"file_system_id": "tenant-aws-us-east-1"`) + describe := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "--region", "aws-us-west-2", "fs", "describe-file-system", "--file-system-id", "tenant-aws-us-west-2") describe.wantExitCode(0) - describe.wantStdoutContains(`"tenant_id": "tenant-scratch"`) + describe.wantStdoutContains(`"file_system_id": "tenant-aws-us-west-2"`) describe.wantStdoutContains(`"region_code": "aws-us-west-2"`) - describe.wantStdoutNotContains("key-scratch") + describe.wantStdoutNotContains("drive9_") callsBeforeMissingSelectorCommands := len(readFakeDrive9Calls(t, recordPath)) for _, args := range [][]string{ {"fs", "list-files", "--path", "/"}, @@ -689,51 +697,51 @@ func TestFSResourceRegistrySelectionAcrossCommandFamilies(t *testing.T) { } { missing := runTIWithInput(t, bin, "", baseEnv, append([]string{"--profile", "stage"}, args...)...) missing.wantExitCode(2) - missing.wantStderrContains("file system name is required; pass --file-system-name or set TI_FS_FILE_SYSTEM_NAME") + missing.wantStderrContains("file system ID is required") } if calls := readFakeDrive9Calls(t, recordPath); len(calls) != callsBeforeMissingSelectorCommands { t.Fatalf("missing resource selection must fail before invoking Drive9: calls before=%d after=%d", callsBeforeMissingSelectorCommands, len(calls)) } missingDryRun := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "fs", "create-directory", "--path", "/tmp", "--dry-run") missingDryRun.wantExitCode(2) - missingDryRun.wantStderrContains("file system name is required; pass --file-system-name or set TI_FS_FILE_SYSTEM_NAME") + missingDryRun.wantStderrContains("file system ID is required") - dataPlane := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "fs", "list-files", "--file-system-name", "scratch", "--path", "/") + dataPlane := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "fs", "list-files", "--file-system-id", "tenant-aws-us-west-2", "--path", "/") dataPlane.wantExitCode(0) - vault := runTIWithInput(t, bin, "", append(baseEnv, "TI_FS_FILE_SYSTEM_NAME=workspace"), "--profile", "stage", "fs-vault", "list-secrets") + vault := runTIWithInput(t, bin, "", append(baseEnv, "TI_FS_FILE_SYSTEM_ID=tenant-aws-us-east-1"), "--profile", "stage", "fs-vault", "list-secrets") vault.wantExitCode(0) - journal := runTIWithInput(t, bin, "", append(baseEnv, "TI_FS_FILE_SYSTEM_NAME=workspace"), "--profile", "stage", "fs-journal", "create-journal", "--file-system-name", "scratch", "--journal-id", "jrn-e2e") + journal := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "fs-journal", "create-journal", "--file-system-id", "tenant-aws-us-west-2", "--journal-id", "jrn-e2e") journal.wantExitCode(0) - git := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "fs-git", "hydrate-git-workspace", "--file-system-name", "scratch", "--target-path", filepath.Join(home, "workspace")) + git := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "fs-git", "hydrate-git-workspace", "--file-system-id", "tenant-aws-us-west-2", "--target-path", filepath.Join(home, "workspace")) git.wantExitCode(0) - mount := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "fs", "mount-file-system", "--file-system-name", "scratch", "--mount-path", filepath.Join(home, "mount"), "--foreground") + mount := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "fs", "mount-file-system", "--file-system-id", "tenant-aws-us-west-2", "--mount-path", filepath.Join(home, "mount"), "--foreground") mount.wantExitCode(0) calls := readFakeDrive9Calls(t, recordPath) - assertFakeDrive9Call(t, calls, []string{"create", "--json", "--name", "workspace"}, "", home, "stage", "workspace", "https://fs-east.test", "aws-us-east-1") - assertFakeDrive9Call(t, calls, []string{"create", "--json", "--name", "scratch"}, "", home, "stage", "scratch", "https://fs-west.test", "aws-us-west-2") - assertFakeDrive9Call(t, calls, []string{"fs", "ls"}, "key-scratch", home, "stage", "scratch", "https://fs-west.test", "aws-us-west-2") - assertFakeDrive9Call(t, calls, []string{"vault", "ls"}, "key-workspace", home, "stage", "workspace", "https://fs-east.test", "aws-us-east-1") - assertFakeDrive9Call(t, calls, []string{"journal", "new"}, "key-scratch", home, "stage", "scratch", "https://fs-west.test", "aws-us-west-2") - assertFakeDrive9Call(t, calls, []string{"git", "hydrate"}, "key-scratch", home, "stage", "scratch", "https://fs-west.test", "aws-us-west-2") - assertFakeDrive9Call(t, calls, []string{"mount"}, "key-scratch", home, "stage", "scratch", "https://fs-west.test", "aws-us-west-2") - - deleteScratch := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "fs", "delete-file-system", "--file-system-name", "scratch") + assertFakeDrive9TransientCall(t, calls, []string{"create"}, "", home, "https://fs-east.test", "aws-us-east-1") + assertFakeDrive9Call(t, calls, []string{"admin", "tenant", "list"}, "", home, "stage", "_control-plane", "https://fs-east.test", "aws-us-east-1") + assertFakeDrive9Call(t, calls, []string{"fs", "ls"}, drive9TestToken("tenant-aws-us-west-2"), home, "stage", "tenant-aws-us-west-2", "https://fs-west.test", "aws-us-west-2") + assertFakeDrive9Call(t, calls, []string{"vault", "ls"}, drive9TestToken("tenant-aws-us-east-1"), home, "stage", "tenant-aws-us-east-1", "https://fs-east.test", "aws-us-east-1") + + deleteScratch := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "--region", "aws-us-west-2", "fs", "delete-file-system", "--file-system-id", "tenant-aws-us-west-2") deleteScratch.wantExitCode(0) deleteScratch.wantStdoutContains(`"status": "deleting"`) afterDelete := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "fs", "list-file-systems") afterDelete.wantExitCode(0) - afterDelete.wantStdoutContains(`"file_system_name": "workspace"`) - afterDelete.wantStdoutNotContains(`"file_system_name": "scratch"`) + afterDelete.wantStdoutContains(`"file_system_id": "tenant-aws-us-east-1"`) + afterDelete.wantStdoutNotContains(`"file_system_id": "tenant-aws-us-west-2"`) + westAfterDelete := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "--region", "aws-us-west-2", "fs", "list-file-systems") + westAfterDelete.wantExitCode(0) + westAfterDelete.wantStdoutNotContains(`"file_system_id": "tenant-aws-us-west-2"`) stillMissingAfterDelete := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "fs", "list-files", "--path", "/") stillMissingAfterDelete.wantExitCode(2) - stillMissingAfterDelete.wantStderrContains("file system name is required; pass --file-system-name or set TI_FS_FILE_SYSTEM_NAME") - assertFakeDrive9Call(t, readFakeDrive9Calls(t, recordPath), []string{"delete", "--json", "--yes"}, "key-scratch", home, "stage", "scratch", "https://fs-west.test", "aws-us-west-2") + stillMissingAfterDelete.wantStderrContains("file system ID is required") + assertFakeDrive9Call(t, readFakeDrive9Calls(t, recordPath), []string{"admin", "tenant", "delete"}, "", home, "stage", "_control-plane", "https://fs-west.test", "aws-us-west-2") for _, args := range [][]string{ {"--profile", "stage", "fs", "set-default-file-system"}, {"--profile", "stage", "fs", "unset-default-file-system"}, - {"--profile", "stage", "fs", "create-file-system", "--file-system-name", "removed-flag", "--set-default"}, + {"--profile", "stage", "fs", "create-file-system", "--set-default"}, } { removed := runTIWithInput(t, bin, "", baseEnv, args...) removed.wantExitCode(2) @@ -766,15 +774,14 @@ func TestFSConfigurationFreeAccess(t *testing.T) { "TI_TEST_FS_MANIFEST_URL=" + manifestServer.URL, } authEnv := append(append([]string{}, baseEnv...), - "TI_FS_FILE_SYSTEM_NAME=workspace", - "TI_FS_TOKEN=configuration-free-token", - "TI_REGION_CODE=aws-us-east-1", - "TIDB_CLOUD_PUBLIC_KEY=must-not-reach-data-plane", + "TDC_FS_TOKEN="+drive9TestToken("tenant-sandbox"), + "TDC_REGION_CODE=aws-us-east-1", + "TDC_PUBLIC_KEY=must-not-reach-data-plane", ) - localList := runTIWithInput(t, bin, "", baseEnv, "fs", "list-file-systems") - localList.wantExitCode(0) - localList.wantStdoutContains(`"file_systems": []`) + remoteList := runTIWithInput(t, bin, "", baseEnv, "fs", "list-file-systems") + remoteList.wantExitCode(2) + remoteList.wantStderrContains("profile \"default\" not found") for _, args := range [][]string{ {"fs", "check-file-system"}, @@ -790,16 +797,14 @@ func TestFSConfigurationFreeAccess(t *testing.T) { flagsOnly := runTIWithInput(t, bin, "", baseEnv, "--region", "aws-us-east-1", "fs", "list-files", - "--file-system-name", "workspace", - "--fs-token", "flag-token", + "--fs-token", drive9TestToken("tenant-flag"), "--path", "/", ) flagsOnly.wantExitCode(0) - mixed := runTIWithInput(t, bin, "", append(baseEnv, "TI_FS_TOKEN=mixed-token"), + mixed := runTIWithInput(t, bin, "", append(baseEnv, "TDC_FS_TOKEN="+drive9TestToken("tenant-mixed")), "--region", "aws-us-east-1", "fs", "list-files", - "--file-system-name", "workspace", "--path", "/", ) mixed.wantExitCode(0) @@ -822,9 +827,10 @@ func TestFSConfigurationFreeAccess(t *testing.T) { unmount.wantExitCode(0) for _, path := range []string{ - filepath.Join(home, ".ti", "config"), - filepath.Join(home, ".ti", "credentials"), - filepath.Join(home, ".ti", "fs_resources"), + filepath.Join(home, ".tdc", "config"), + filepath.Join(home, ".tdc", "credentials"), + filepath.Join(home, ".tdc", "fs_resources"), + filepath.Join(home, ".tdc", "fs_credentials"), } { if _, err := os.Stat(path); !os.IsNotExist(err) { t.Fatalf("configuration-free command persisted ti configuration at %s: %v", path, err) @@ -834,7 +840,7 @@ func TestFSConfigurationFreeAccess(t *testing.T) { if err != nil { t.Fatalf("read configuration-free operation log: %v", err) } - for _, secret := range []string{"configuration-free-token", "flag-token", "mixed-token", "must-not-reach-data-plane"} { + for _, secret := range []string{drive9TestToken("tenant-sandbox"), drive9TestToken("tenant-flag"), drive9TestToken("tenant-mixed"), "must-not-reach-data-plane"} { if strings.Contains(string(logData), secret) { t.Fatalf("configuration-free operation log leaked a credential") } @@ -848,9 +854,94 @@ func TestFSConfigurationFreeAccess(t *testing.T) { t.Fatalf("data-plane companion inherited TiDB Cloud or raw ti secrets: %#v", call) } } - assertFakeDrive9Call(t, calls, []string{"fs", "ls"}, "configuration-free-token", home, "default", "workspace", "https://fs-east.test", "aws-us-east-1") - assertFakeDrive9Call(t, calls, []string{"mount", "drain"}, "", home, "default", "workspace", "https://fs-east.test", "aws-us-east-1") - assertFakeDrive9Call(t, calls, []string{"umount"}, "", home, "default", "workspace", "https://fs-east.test", "aws-us-east-1") + assertFakeDrive9Call(t, calls, []string{"fs", "ls"}, drive9TestToken("tenant-sandbox"), home, "default", "tenant-sandbox", "https://fs-east.test", "aws-us-east-1") + assertFakeDrive9Call(t, calls, []string{"mount", "drain"}, "", home, "default", "tenant-sandbox", "https://fs-east.test", "aws-us-east-1") + assertFakeDrive9Call(t, calls, []string{"umount"}, "", home, "default", "tenant-sandbox", "https://fs-east.test", "aws-us-east-1") +} + +func TestFSImportFileSystemToken(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX token-file permission checks are not available on Windows") + } + bin := tiBinary(t) + home := t.TempDir() + companion := filepath.Join(t.TempDir(), "tdc-drive9") + build := exec.Command("go", "build", "-o", companion, "./testdata/fake-drive9.go") + build.Dir = "." + if output, err := build.CombinedOutput(); err != nil { + t.Fatalf("build fake Drive9 companion: %v\n%s", err, output) + } + token := drive9TestToken("tenant-imported") + manifestServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprint(w, `{"service":"drive9","regions":[{"region_code":"aws-us-east-1","mode":"tidb_cloud_native","server_url":"https://fs.test","cloud_provider":"aws","tidb_region":"us-east-1"}]}`) + })) + defer manifestServer.Close() + recordPath := filepath.Join(t.TempDir(), "calls.jsonl") + baseEnv := []string{ + "HOME=" + home, + "TI_DRIVE9_BIN=" + companion, + "FAKE_DRIVE9_RECORD=" + recordPath, + "FAKE_DRIVE9_EXPECT_API_KEY=" + token, + "TI_ALLOW_TEST_ENDPOINTS=1", + "TI_TEST_FS_MANIFEST_URL=" + manifestServer.URL, + } + tokenPath := filepath.Join(t.TempDir(), "fs-token") + if err := os.WriteFile(tokenPath, []byte(token+"\n"), 0o600); err != nil { + t.Fatal(err) + } + + dryHome := t.TempDir() + dryRun := runTIWithInput(t, bin, "", append(append([]string{}, baseEnv...), "HOME="+dryHome), + "--region", "aws-us-east-1", "fs", "import-file-system-token", "--from-file", tokenPath, "--dry-run") + dryRun.wantExitCode(0) + if _, err := fscred.GetCredential(dryHome, "default", "tenant-imported"); err == nil { + t.Fatal("dry-run persisted imported credentials") + } + + imported := runTIWithInput(t, bin, "", baseEnv, + "--region", "aws-us-east-1", "fs", "import-file-system-token", "--from-file", tokenPath) + imported.wantExitCode(0) + imported.wantStdoutContains(`"file_system_id": "tenant-imported"`) + imported.wantStdoutNotContains(token) + credential, err := fscred.GetCredential(home, "default", "tenant-imported") + if err != nil || credential.APIKey != token || credential.RegionCode != "aws-us-east-1" { + t.Fatalf("imported credential=%#v err=%v", credential, err) + } + + useStored := runTIWithInput(t, bin, "", baseEnv, + "fs", "list-files", "--file-system-id", "tenant-imported", "--path", "/") + useStored.wantExitCode(0) + calls := readFakeDrive9Calls(t, recordPath) + assertFakeDrive9Call(t, calls, []string{"fs", "ls"}, token, home, "default", "tenant-imported", "https://fs.test", "aws-us-east-1") + + multipleSources := runTIWithInput(t, bin, "", append(baseEnv, "TDC_FS_TOKEN="+token), + "--region", "aws-us-east-1", "fs", "import-file-system-token", "--from-file", tokenPath) + multipleSources.wantExitCode(2) + multipleSources.wantStderrContains("provide exactly one") + + insecurePath := filepath.Join(t.TempDir(), "insecure-token") + if err := os.WriteFile(insecurePath, []byte(token), 0o644); err != nil { + t.Fatal(err) + } + insecure := runTIWithInput(t, bin, "", baseEnv, + "--region", "aws-us-east-1", "fs", "import-file-system-token", "--from-file", insecurePath) + insecure.wantExitCode(2) + insecure.wantStderrContains("mode 0600 or stricter") + + stdinHome := t.TempDir() + stdinImport := runTIWithInput(t, bin, token+"\n", append(append([]string{}, baseEnv...), "HOME="+stdinHome), + "--region", "aws-us-east-1", "fs", "import-file-system-token", "--from-file", "-") + stdinImport.wantExitCode(0) + if _, err := fscred.GetCredential(stdinHome, "default", "tenant-imported"); err != nil { + t.Fatalf("stdin import did not store credentials: %v", err) + } +} + +func drive9TestToken(fileSystemID string) string { + header := base64.RawURLEncoding.EncodeToString([]byte(`{"alg":"none"}`)) + payload, _ := json.Marshal(map[string]string{"tenant_id": fileSystemID}) + jwt := header + "." + base64.RawURLEncoding.EncodeToString(payload) + ".signature" + return "drive9_" + base64.RawURLEncoding.EncodeToString([]byte(jwt)) } type fakeDrive9Call struct { @@ -911,6 +1002,36 @@ func assertFakeDrive9Call(t *testing.T, calls []fakeDrive9Call, prefix []string, t.Fatalf("missing fake Drive9 call with prefix %v in %#v", prefix, calls) } +func assertFakeDrive9TransientCall(t *testing.T, calls []fakeDrive9Call, prefix []string, apiKey, persistentHome, server, regionCode string) { + t.Helper() + for _, call := range calls { + if len(call.Args) < len(prefix) { + continue + } + matches := true + for i := range prefix { + if call.Args[i] != prefix[i] { + matches = false + break + } + } + if !matches { + continue + } + if call.APIKey != apiKey || call.Server != server || call.RegionCode != regionCode { + t.Fatalf("unexpected fake Drive9 environment for %v: %#v", prefix, call) + } + if call.Home == "" || strings.HasPrefix(call.Home, filepath.Join(persistentHome, ".tdc")) { + t.Fatalf("fake Drive9 call %v used persistent HOME %q", prefix, call.Home) + } + if _, err := os.Stat(call.Home); !os.IsNotExist(err) { + t.Fatalf("transient Drive9 HOME still exists: %q, err=%v", call.Home, err) + } + return + } + t.Fatalf("missing fake Drive9 call with prefix %v in %#v", prefix, calls) +} + func tiBinary(t *testing.T) string { t.Helper() bin := os.Getenv("TI_E2E_BIN") @@ -1194,15 +1315,14 @@ func runTIUsingLoggingSettings(t *testing.T, bin string, env []string, args ...s func runTIProcess(t *testing.T, bin, stdin string, env []string, disableLoggingByDefault bool, args ...string) commandResult { t.Helper() - if os.Getenv("TI_LIVE") != "1" && !envContains(env, "HOME") { - env = append(env, "HOME="+t.TempDir()) - } - if os.Getenv("TI_LIVE") == "1" && hasLiveFSCommandFamily(args) && !envContains(env, "TI_FS_FILE_SYSTEM_NAME") { - name := strings.TrimSpace(os.Getenv("TI_LIVE_FS_NAME")) - if name == "" { - name = "workspace" + if os.Getenv("TI_LIVE") == "1" && hasLiveFSCommandFamily(args) && !envContains(env, "TI_FS_FILE_SYSTEM_ID") && !envContains(env, "TI_FS_TOKEN") && !envContains(env, "TDC_FS_TOKEN") { + fileSystemID := strings.TrimSpace(os.Getenv("TI_LIVE_FS_ID")) + if fileSystemID == "" { + fileSystemID = liveFSSelectedID + } + if fileSystemID != "" { + env = append(env, "TI_FS_FILE_SYSTEM_ID="+fileSystemID) } - env = append(env, "TI_FS_FILE_SYSTEM_NAME="+name) } cmd := exec.Command(bin, args...) diff --git a/e2e/live_test.go b/e2e/live_test.go index 0cdd267..b0852a8 100644 --- a/e2e/live_test.go +++ b/e2e/live_test.go @@ -17,7 +17,6 @@ import ( "github.com/tidbcloud/ti-cli/internal/api" "github.com/tidbcloud/ti-cli/internal/api/endpoints" apifs "github.com/tidbcloud/ti-cli/internal/api/fs" - "github.com/tidbcloud/ti-cli/internal/apperr" "github.com/tidbcloud/ti-cli/internal/auth" "github.com/tidbcloud/ti-cli/internal/authz" "github.com/tidbcloud/ti-cli/internal/config" @@ -27,14 +26,15 @@ import ( const defaultLiveProfile = "live-e2e" var ( - liveFSResourceMu sync.Mutex - liveFSResourceAutoCreated bool - liveProfileConfigureMu sync.Mutex + liveFSResourceMu sync.Mutex + liveFSResourceAutoCreatedID string + liveFSSelectedID string + liveProfileConfigureMu sync.Mutex ) func TestMain(m *testing.M) { code := m.Run() - if liveFSResourceAutoCreated { + if liveFSResourceAutoCreatedID != "" { cleanupAutoCreatedLiveFSResource() } os.Exit(code) @@ -83,82 +83,50 @@ func TestLiveOrganizationAPIReadOnlyProbes(t *testing.T) { liveGETJSON(t, iam, "/v1beta1/projects") } -func TestLiveFSResourceRegistryLifecycle(t *testing.T) { +func TestLiveFSRemoteInventoryLifecycle(t *testing.T) { requireLive(t) bin := tiBinary(t) profileName := liveProfileName(t) - suffix := fmt.Sprintf("%s-%d", time.Now().UTC().Format("20060102150405"), os.Getpid()) - names := []string{"ti-e2e-fs-" + suffix + "-a", "ti-e2e-fs-" + suffix + "-b"} - created := make(map[string]bool, len(names)) + preflightList := runTI(t, bin, "--profile", profileName, "fs", "list-file-systems") + preflightList.wantExitCode(0) + create := runTI(t, bin, "--profile", profileName, "fs", "create-file-system", "--wait") + if create.exitCode != 0 && isLiveFSQuotaError(create.stderr) { + t.Skipf("tdc fs live inventory lifecycle requires one free Starter slot: %s", strings.TrimSpace(create.stderr)) + } + create.wantExitCode(0) + var created struct { + FileSystemID string `json:"file_system_id"` + FSToken string `json:"fs_token"` + } + if err := json.Unmarshal([]byte(create.stdout), &created); err != nil || created.FileSystemID == "" || created.FSToken == "" { + t.Fatalf("decode live tdc fs create result: %v", err) + } defer func() { - for i := len(names) - 1; i >= 0; i-- { - name := names[i] - if !created[name] { - continue - } - result := runTI(t, bin, "--profile", profileName, "fs", "delete-file-system", "--file-system-name", name) - if result.exitCode != 0 { - t.Logf("cleanup delete failed for ti fs resource %q: exit=%d stdout=%s stderr=%s", name, result.exitCode, result.stdout, result.stderr) - } + result := runTI(t, bin, "--profile", profileName, "fs", "delete-file-system", "--file-system-id", created.FileSystemID) + if result.exitCode != 0 { + t.Logf("cleanup delete failed for tdc fs resource %q: exit=%d stdout=%s stderr=%s", created.FileSystemID, result.exitCode, result.stdout, result.stderr) } }() - for i, name := range names { - create := runTI(t, bin, "--profile", profileName, "fs", "create-file-system", "--file-system-name", name, "--wait") - if create.exitCode != 0 { - if isLiveFSQuotaError(create.stderr) { - if i == 0 { - t.Skipf("ti fs live registry lifecycle requires one free Starter slot: %s", strings.TrimSpace(create.stderr)) - } - t.Logf("second ti fs resource could not be created because Starter quota is full; single-resource live flow completed and multi-resource selection remains covered by the fake-companion e2e: %s", strings.TrimSpace(create.stderr)) - check := runTI(t, bin, "--profile", profileName, "fs", "check-file-system", "--file-system-name", names[0]) - check.wantExitCode(0) - check.wantStdoutContains(`"status": "passed"`) - return - } - create.fail("create live ti fs registry resource") - } - if strings.Contains(create.stdout, `"status": "exists"`) { - create.fail("generated live ti fs resource name already existed; refusing to delete a resource not created by this test") - } - created[name] = true - create.wantStdoutContains(`"credentials_stored": true`) - create.wantStdoutContains(`"status": "ready"`) - } - list := runTI(t, bin, "--profile", profileName, "fs", "list-file-systems") list.wantExitCode(0) - for _, name := range names { - list.wantStdoutContains(`"file_system_name": "` + name + `"`) - } - list.wantStdoutNotContains("default_file_system_name") - list.wantStdoutNotContains("is_default") + list.wantStdoutContains(`"file_system_id": "` + created.FileSystemID + `"`) + list.wantStdoutContains(`"has_local_token": true`) + list.wantStdoutNotContains(created.FSToken) - missingSelector := runTIWithInput(t, bin, "", []string{"TI_FS_FILE_SYSTEM_NAME="}, "--profile", profileName, "fs", "check-file-system") + missingSelector := runTIWithInput(t, bin, "", []string{"TI_FS_FILE_SYSTEM_ID="}, "--profile", profileName, "fs", "check-file-system") missingSelector.wantExitCode(2) - missingSelector.wantStderrContains("file system name is required; pass --file-system-name or set TI_FS_FILE_SYSTEM_NAME") - environmentCheck := runTIWithInput(t, bin, "", []string{"TI_FS_FILE_SYSTEM_NAME=" + names[0]}, "--profile", profileName, "fs", "check-file-system") + missingSelector.wantStderrContains("file system ID is required") + environmentCheck := runTIWithInput(t, bin, "", []string{"TI_FS_FILE_SYSTEM_ID=" + created.FileSystemID}, "--profile", profileName, "fs", "check-file-system") environmentCheck.wantExitCode(0) - environmentCheck.wantStdoutContains(`"file_system_name": "` + names[0] + `"`) - explicitCheck := runTI(t, bin, "--profile", profileName, "fs", "check-file-system", "--file-system-name", names[1]) + environmentCheck.wantStdoutContains(`"file_system_id": "` + created.FileSystemID + `"`) + explicitCheck := runTI(t, bin, "--profile", profileName, "fs", "check-file-system", "--file-system-id", created.FileSystemID) explicitCheck.wantExitCode(0) - explicitCheck.wantStdoutContains(`"file_system_name": "` + names[1] + `"`) - - deleteFirst := runTI(t, bin, "--profile", profileName, "fs", "delete-file-system", "--file-system-name", names[0]) - deleteFirst.wantExitCode(0) - deleteFirst.wantStdoutContains(`"status": "deleting"`) - deleteFirst.wantStdoutContains(`"remote_deletion_state": "deleting"`) - created[names[0]] = false - remaining := runTI(t, bin, "--profile", profileName, "fs", "describe-file-system", "--file-system-name", names[1]) - remaining.wantExitCode(0) - remaining.wantStdoutContains(`"file_system_name": "` + names[1] + `"`) - - deleteSecond := runTI(t, bin, "--profile", profileName, "fs", "delete-file-system", "--file-system-name", names[1]) - deleteSecond.wantExitCode(0) - deleteSecond.wantStdoutContains(`"status": "deleting"`) - deleteSecond.wantStdoutContains(`"remote_deletion_state": "deleting"`) - created[names[1]] = false + explicitCheck.wantStdoutContains(`"file_system_id": "` + created.FileSystemID + `"`) + describe := runTI(t, bin, "--profile", profileName, "fs", "describe-file-system", "--file-system-id", created.FileSystemID) + describe.wantExitCode(0) + describe.wantStdoutContains(`"file_system_id": "` + created.FileSystemID + `"`) } func TestLiveCLICommandSurface(t *testing.T) { @@ -273,11 +241,10 @@ func TestLiveFSCommandSurface(t *testing.T) { requireLive(t) bin := tiBinary(t) profileName := liveProfileName(t) - fileSystemName := liveFileSystemName(t) - ensureLiveFSResource(t, bin, profileName) + selected := ensureLiveFSResource(t, bin, profileName) testLiveHelpCommands(t, bin, [][]string{ {"fs", "help"}, - {"fs", "create-file-system", "help"}, {"fs", "list-file-systems", "help"}, + {"fs", "create-file-system", "help"}, {"fs", "import-file-system-token", "help"}, {"fs", "list-file-systems", "help"}, {"fs", "describe-file-system", "help"}, {"fs", "copy-file", "help"}, {"fs", "read-file", "help"}, {"fs", "chmod-file", "help"}, {"fs", "create-symlink", "help"}, {"fs", "create-hardlink", "help"}, @@ -293,8 +260,8 @@ func TestLiveFSCommandSurface(t *testing.T) { {"fs", "mount", "help"}, {"fs", "drain", "help"}, {"fs", "umount", "help"}, }) testLiveMutatingDryRuns(t, bin, profileName, [][]string{ - {"fs", "create-file-system", "--file-system-name", fileSystemName, "--wait"}, - {"fs", "delete-file-system", "--file-system-name", fileSystemName}, + {"fs", "create-file-system", "--wait"}, + {"fs", "delete-file-system", "--file-system-id", selected.FSTenantID}, {"fs", "create-layer", "--layer-id", "layer-1", "--base-root-path", "/workspace", "--layer-name", "dev"}, {"fs", "create-layer-checkpoint", "--layer-id", "layer-1", "--checkpoint-id", "cp-1"}, {"fs", "rollback-layer", "--layer-id", "layer-1"}, {"fs", "commit-layer", "--layer-id", "layer-1"}, @@ -319,7 +286,7 @@ func TestLiveFSCommandSurface(t *testing.T) { for _, args := range [][]string{ {"fs", "set-default-file-system"}, {"fs", "unset-default-file-system"}, - {"fs", "create-file-system", "--file-system-name", "removed-flag", "--set-default"}, + {"fs", "create-file-system", "--set-default"}, } { result := runTI(t, bin, append([]string{"--profile", profileName}, args...)...) result.wantExitCode(2) @@ -507,15 +474,15 @@ func testLiveReadOnlyDryRunRejections(t *testing.T, bin, profileName string, com } } -func resolveLiveFSResource(t *testing.T, profile *config.Profile, name string) *config.Profile { +func resolveLiveFSResourceByID(t *testing.T, profile *config.Profile, fileSystemID string) *config.Profile { t.Helper() home, err := os.UserHomeDir() if err != nil { t.Fatalf("determine home directory: %v", err) } - selected, _, err := fscred.Resolve(home, profile, name, true, nil) + selected, _, err := fscred.ResolveCredential(home, profile, fscred.ResolveCredentialOptions{FileSystemID: fileSystemID, FileSystemIDExplicit: true, TokenRequired: true}) if err != nil { - t.Fatalf("resolve live ti fs resource %q: %v", name, err) + t.Fatalf("resolve live tdc fs resource %q: %v", fileSystemID, err) } return selected } @@ -583,7 +550,7 @@ func TestLiveFSDataPlaneLifecycle(t *testing.T) { rangeRead := runTI(t, bin, "--profile", profileName, "fs", "read-file", "--path", sourcePath, "--offset", "6", "--length", "3") rangeRead.wantExitCode(0) - if rangeRead.stdout != "ti" { + if rangeRead.stdout != "ti " { rangeRead.fail("read-file --offset/--length should return the requested byte range") } @@ -1229,25 +1196,23 @@ func TestLiveFSConfigurationFreeAccess(t *testing.T) { bin := tiBinary(t) profileName := liveProfileName(t) suffix := fmt.Sprintf("%s-%d", time.Now().UTC().Format("20060102150405"), os.Getpid()) - fileSystemName := "ti-e2e-token-" + suffix - create := runTI(t, bin, "--profile", profileName, "fs", "create-file-system", "--file-system-name", fileSystemName, "--wait") + preflightList := runTI(t, bin, "--profile", profileName, "fs", "list-file-systems") + preflightList.wantExitCode(0) + create := runTI(t, bin, "--profile", profileName, "fs", "create-file-system", "--wait") create.wantExitCode(0) var created struct { - FileSystemName string `json:"file_system_name"` - RegionCode string `json:"region_code"` - FSToken string `json:"fs_token"` - Status string `json:"status"` + FileSystemID string `json:"file_system_id"` + RegionCode string `json:"region_code"` + FSToken string `json:"fs_token"` + Status string `json:"status"` } if err := json.Unmarshal([]byte(create.stdout), &created); err != nil { t.Fatalf("decode configuration-free FS create result: %v", err) } create.stdout = "" - if created.FileSystemName != fileSystemName || created.RegionCode == "" || created.FSToken == "" { + if created.FileSystemID == "" || created.RegionCode == "" || created.FSToken == "" { t.Fatalf("configuration-free FS create result is incomplete") } - if created.Status == "exists" { - t.Fatalf("generated configuration-free FS resource name unexpectedly existed") - } if created.Status != "ready" { t.Fatalf("--wait returned ti fs resource in status %q", created.Status) } @@ -1257,14 +1222,14 @@ func TestLiveFSConfigurationFreeAccess(t *testing.T) { if deletedResource { return } - cleanup := runTI(t, bin, "--profile", profileName, "fs", "delete-file-system", "--file-system-name", fileSystemName) + cleanup := runTI(t, bin, "--profile", profileName, "fs", "delete-file-system", "--file-system-id", created.FileSystemID) if cleanup.exitCode != 0 { - t.Logf("cleanup configuration-free FS resource failed for %q: exit=%d stderr=%s", fileSystemName, cleanup.exitCode, strings.TrimSpace(cleanup.stderr)) + t.Logf("cleanup configuration-free FS resource failed for %q: exit=%d stderr=%s", created.FileSystemID, cleanup.exitCode, strings.TrimSpace(cleanup.stderr)) } }() profile := liveProfile(t) - selected := resolveLiveFSResource(t, profile, fileSystemName) + selected := resolveLiveFSResourceByID(t, profile, created.FileSystemID) if selected.FSAPIKey != created.FSToken || selected.FSPlacementRegionCode != created.RegionCode { t.Fatal("stored FS resource credentials or placement differ from create output") } @@ -1276,7 +1241,7 @@ func TestLiveFSConfigurationFreeAccess(t *testing.T) { if remoteDeleted { return } - cleanup := runTI(t, bin, "--profile", profileName, "fs", "delete-file", "--file-system-name", fileSystemName, "--path", remoteRoot, "--recursive") + cleanup := runTI(t, bin, "--profile", profileName, "fs", "delete-file", "--file-system-id", created.FileSystemID, "--path", remoteRoot, "--recursive") if cleanup.exitCode != 0 && cleanup.exitCode != 5 { t.Logf("cleanup configuration-free remote path failed for %s: exit=%d stderr=%s", remoteRoot, cleanup.exitCode, strings.TrimSpace(cleanup.stderr)) } @@ -1344,6 +1309,7 @@ func TestLiveFSConfigurationFreeAccess(t *testing.T) { filepath.Join(cleanHome, ".ti", "config"), filepath.Join(cleanHome, ".ti", "credentials"), filepath.Join(cleanHome, ".ti", "fs_resources"), + filepath.Join(cleanHome, ".ti", "fs_credentials"), } { if _, err := os.Stat(path); !os.IsNotExist(err) { t.Fatalf("configuration-free live command persisted ti configuration at %s: %v", path, err) @@ -1357,14 +1323,69 @@ func TestLiveFSConfigurationFreeAccess(t *testing.T) { t.Fatalf("successful configuration-free unmount left %d mount locator(s)", len(locators)) } - deleteRemote := runTI(t, bin, "--profile", profileName, "fs", "delete-file", "--file-system-name", fileSystemName, "--path", remoteRoot, "--recursive") + controlHome := t.TempDir() + controlEnv := []string{ + "HOME=" + controlHome, + "TI_PROFILE=", + "TIDB_CLOUD_PUBLIC_KEY=" + profile.TiDBCloudPublicKey, + "TIDB_CLOUD_PRIVATE_KEY=" + profile.TiDBCloudPrivateKey, + "TI_REGION_CODE=" + created.RegionCode, + "TI_FS_FILE_SYSTEM_ID=", + "TI_FS_TOKEN=", + } + cleanList := runTIWithInput(t, bin, "", controlEnv, "fs", "list-file-systems") + cleanList.wantExitCode(0) + cleanList.wantStdoutContains(`"file_system_id": "` + created.FileSystemID + `"`) + cleanDescribe := runTIWithInput(t, bin, "", controlEnv, "fs", "describe-file-system", "--file-system-id", created.FileSystemID) + cleanDescribe.wantExitCode(0) + cleanDescribe.wantStdoutContains(`"has_local_token": false`) + + tokenPath := filepath.Join(t.TempDir(), "fs-token") + if err := os.WriteFile(tokenPath, []byte(created.FSToken+"\n"), 0o600); err != nil { + t.Fatalf("write configuration-free import token: %v", err) + } + importEnv := []string{ + "HOME=" + cleanHome, + "TI_PROFILE=", + "TIDB_CLOUD_PUBLIC_KEY=", + "TIDB_CLOUD_PRIVATE_KEY=", + "TI_REGION_CODE=" + created.RegionCode, + "TI_FS_FILE_SYSTEM_ID=", + "TI_FS_TOKEN=", + } + imported := runTIWithInput(t, bin, "", importEnv, "fs", "import-file-system-token", "--from-file", tokenPath) + imported.wantExitCode(0) + imported.wantStdoutContains(`"file_system_id": "` + created.FileSystemID + `"`) + storedCredentialEnv := []string{ + "HOME=" + cleanHome, + "TI_PROFILE=", + "TIDB_CLOUD_PUBLIC_KEY=", + "TIDB_CLOUD_PRIVATE_KEY=", + "TI_REGION_CODE=", + "TI_FS_FILE_SYSTEM_ID=" + created.FileSystemID, + "TI_FS_TOKEN=", + } + readWithImportedToken := runTIWithInput(t, bin, "", storedCredentialEnv, "fs", "read-file", "--path", remoteRoot+"/seed.txt") + readWithImportedToken.wantExitCode(0) + if readWithImportedToken.stdout != seedContent { + readWithImportedToken.fail("imported local credential should authorize data-plane access") + } + if removed, err := fscred.DeleteCredential(cleanHome, config.DefaultProfile, created.FileSystemID); err != nil || !removed { + t.Fatalf("remove imported credential before control-plane delete: removed=%t err=%v", removed, err) + } + + deleteRemote := runTI(t, bin, "--profile", profileName, "fs", "delete-file", "--file-system-id", created.FileSystemID, "--path", remoteRoot, "--recursive") deleteRemote.wantExitCode(0) remoteDeleted = true - deleteResource := runTI(t, bin, "--profile", profileName, "fs", "delete-file-system", "--file-system-name", fileSystemName) + deleteResource := runTIWithInput(t, bin, "", controlEnv, "fs", "delete-file-system", "--file-system-id", created.FileSystemID) deleteResource.wantExitCode(0) deleteResource.wantStdoutContains(`"status": "deleting"`) deleteResource.wantStdoutContains(`"remote_deletion_state": "deleting"`) deletedResource = true + if _, err := fscred.DeleteCredential(profile.HomeDir, profileName, created.FileSystemID); err != nil { + t.Fatalf("remove original local credential after remote deletion acceptance: %v", err) + } + waitLiveFSInventoryAbsent(t, bin, controlEnv, created.FileSystemID, 2*time.Minute) } func TestLiveFSWebDAVMountRuntime(t *testing.T) { @@ -1839,12 +1860,43 @@ func liveFSTokenEnv(profile *config.Profile, home string) []string { "TI_PROFILE=", "TIDB_CLOUD_PUBLIC_KEY=", "TIDB_CLOUD_PRIVATE_KEY=", - "TI_FS_FILE_SYSTEM_NAME=" + profile.FSResourceName, + "TI_FS_FILE_SYSTEM_ID=", "TI_FS_TOKEN=" + profile.FSAPIKey, "TI_REGION_CODE=" + profile.FSPlacementRegionCode, } } +func waitLiveFSInventoryAbsent(t *testing.T, bin string, env []string, fileSystemID string, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for { + result := runTIWithInput(t, bin, "", env, "fs", "list-file-systems") + result.wantExitCode(0) + var inventory struct { + FileSystems []struct { + FileSystemID string `json:"file_system_id"` + } `json:"file_systems"` + } + if err := json.Unmarshal([]byte(result.stdout), &inventory); err != nil { + t.Fatalf("decode post-delete live FS inventory: %v", err) + } + found := false + for _, resource := range inventory.FileSystems { + if resource.FileSystemID == fileSystemID { + found = true + break + } + } + if !found { + return + } + if time.Now().After(deadline) { + t.Fatalf("file system %q remained in remote inventory after deletion timeout", fileSystemID) + } + time.Sleep(2 * time.Second) + } +} + func liveFSLocatorEnv(home string) []string { return []string{ "HOME=" + home, @@ -1852,7 +1904,7 @@ func liveFSLocatorEnv(home string) []string { "TI_PROFILE=", "TIDB_CLOUD_PUBLIC_KEY=", "TIDB_CLOUD_PRIVATE_KEY=", - "TI_FS_FILE_SYSTEM_NAME=", + "TI_FS_FILE_SYSTEM_ID=", "TI_FS_TOKEN=", "TI_REGION_CODE=", } @@ -1912,28 +1964,47 @@ func ensureLiveFSResource(t *testing.T, bin, profileName string) *config.Profile if err != nil { t.Fatalf("determine home directory: %v", err) } - if err := fscred.MigrateLegacy(home, profile); err != nil { + if err := fscred.MigrateNameRegistry(home, profile); err != nil { t.Fatalf("migrate live fs resource: %v", err) } - name := liveFileSystemName(t) - if selected, _, err := fscred.Resolve(home, profile, name, true, nil); err == nil { + requestedID := strings.TrimSpace(os.Getenv("TI_LIVE_FS_ID")) + list := runTI(t, bin, "--profile", profileName, "fs", "list-file-systems") + list.wantExitCode(0) + var inventory struct { + FileSystems []struct { + FileSystemID string `json:"file_system_id"` + HasLocalToken bool `json:"has_local_token"` + } `json:"file_systems"` + } + if err := json.Unmarshal([]byte(list.stdout), &inventory); err != nil { + t.Fatalf("decode live fs inventory: %v", err) + } + for _, resource := range inventory.FileSystems { + if !resource.HasLocalToken || (requestedID != "" && requestedID != resource.FileSystemID) { + continue + } + selected := resolveLiveFSResourceByID(t, profile, resource.FileSystemID) + liveFSSelectedID = resource.FileSystemID waitLiveFSReady(t, bin, profileName, selected, 10*time.Minute) return selected - } else if apperr.CodeFor(err) != "fs.resource_not_found" { - t.Fatalf("resolve live fs resource %q: %v", name, err) } - create := runTI(t, bin, "--profile", profileName, "fs", "create-file-system", "--file-system-name", name, "--wait") + if requestedID != "" { + t.Fatalf("TI_LIVE_FS_ID %q is not remotely visible with a local token", requestedID) + } + create := runTI(t, bin, "--profile", profileName, "fs", "create-file-system", "--wait") create.wantExitCode(0) create.wantStdoutContains(`"credentials_stored": true`) create.wantStdoutContains(`"status": "ready"`) - liveFSResourceAutoCreated = true - - profile = liveProfile(t) - selected, _, err := fscred.Resolve(home, profile, name, true, nil) - if err != nil { - t.Fatalf("ti fs resource %q was created but is not in profile %q registry: %v", name, profileName, err) + var created struct { + FileSystemID string `json:"file_system_id"` + } + if err := json.Unmarshal([]byte(create.stdout), &created); err != nil || created.FileSystemID == "" { + t.Fatalf("decode created live fs resource: %v", err) } + liveFSResourceAutoCreatedID = created.FileSystemID + liveFSSelectedID = created.FileSystemID + selected := resolveLiveFSResourceByID(t, profile, created.FileSystemID) return selected } @@ -1946,7 +2017,7 @@ func waitLiveFSReady(t *testing.T, bin, profileName string, profile *config.Prof } probeRemotePath := fmt.Sprintf("/ti-e2e-readiness-%d-%d.txt", os.Getpid(), time.Now().UnixNano()) defer func() { - cleanup := runLiveFSSetupCommand(t, bin, "--profile", profileName, "fs", "delete-file", "--file-system-name", profile.FSResourceName, "--path", probeRemotePath) + cleanup := runLiveFSSetupCommand(t, bin, "--profile", profileName, "fs", "delete-file", "--file-system-id", profile.FSTenantID, "--path", probeRemotePath) if cleanup.exitCode != 0 && !isLiveFSNotFound(cleanup.stderr) { t.Logf("cleanup ti fs readiness probe failed: exit=%d stderr=%s", cleanup.exitCode, strings.TrimSpace(cleanup.stderr)) } @@ -1962,9 +2033,9 @@ func waitLiveFSReady(t *testing.T, bin, profileName string, profile *config.Prof lastStatus = status state := strings.ToLower(strings.TrimSpace(status.Status)) if state == "" || (!strings.Contains(state, "provision") && !strings.Contains(state, "delet")) { - lastProbe = runTI(t, bin, "--profile", profileName, "fs", "copy-file", "--file-system-name", profile.FSResourceName, "--from-local", probeLocalPath, "--to-remote", probeRemotePath, "--overwrite") + lastProbe = runTI(t, bin, "--profile", profileName, "fs", "copy-file", "--file-system-id", profile.FSTenantID, "--from-local", probeLocalPath, "--to-remote", probeRemotePath, "--overwrite") if lastProbe.exitCode == 0 { - cleanup := runLiveFSSetupCommand(t, bin, "--profile", profileName, "fs", "delete-file", "--file-system-name", profile.FSResourceName, "--path", probeRemotePath) + cleanup := runLiveFSSetupCommand(t, bin, "--profile", profileName, "fs", "delete-file", "--file-system-id", profile.FSTenantID, "--path", probeRemotePath) if cleanup.exitCode != 0 && !isLiveFSNotFound(cleanup.stderr) { cleanup.fail("delete ti fs readiness probe") } @@ -1986,7 +2057,7 @@ func waitLiveFSReady(t *testing.T, bin, profileName string, profile *config.Prof } } if time.Now().After(deadline) { - t.Fatalf("timed out waiting for ti fs resource %q in profile %q to become data-plane ready; last_status=%#v last_error=%v last_probe_stderr=%q", profile.FSResourceName, profile.Name, lastStatus, lastErr, strings.TrimSpace(lastProbe.stderr)) + t.Fatalf("timed out waiting for tdc fs resource %q in profile %q to become data-plane ready; last_status=%#v last_error=%v last_probe_stderr=%q", profile.FSTenantID, profile.Name, lastStatus, lastErr, strings.TrimSpace(lastProbe.stderr)) } time.Sleep(5 * time.Second) } @@ -2055,16 +2126,16 @@ func cleanupAutoCreatedLiveFSResource() { return } profileName := liveProfileNameFromEnv() - name := liveFileSystemNameFromEnv() + fileSystemID := liveFSResourceAutoCreatedID cmd := exec.Command( bin, "--profile", profileName, "fs", "delete-file-system", - "--file-system-name", name, + "--file-system-id", fileSystemID, ) output, err := cmd.CombinedOutput() if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "ti live e2e cleanup warning: delete ti fs resource %q failed: %v\n%s", name, err, string(output)) + _, _ = fmt.Fprintf(os.Stderr, "tdc live e2e cleanup warning: delete tdc fs resource %q failed: %v\n%s", fileSystemID, err, string(output)) } } @@ -2072,19 +2143,20 @@ func releaseAutoCreatedLiveFSResource(t *testing.T, bin, profileName string) { t.Helper() liveFSResourceMu.Lock() defer liveFSResourceMu.Unlock() - if !liveFSResourceAutoCreated { + if liveFSResourceAutoCreatedID == "" { return } - name := liveFileSystemName(t) + fileSystemID := liveFSResourceAutoCreatedID result := runTI( t, bin, "--profile", profileName, "fs", "delete-file-system", - "--file-system-name", name, + "--file-system-id", fileSystemID, ) result.wantExitCode(0) - liveFSResourceAutoCreated = false + liveFSResourceAutoCreatedID = "" + liveFSSelectedID = "" } func liveProfileName(t *testing.T) string { @@ -2104,19 +2176,6 @@ func liveProfileNameFromEnv() string { return profileName } -func liveFileSystemName(t *testing.T) string { - t.Helper() - return liveFileSystemNameFromEnv() -} - -func liveFileSystemNameFromEnv() string { - name := strings.TrimSpace(os.Getenv("TI_LIVE_FS_NAME")) - if name == "" { - name = "workspace" - } - return name -} - func liveProfile(t *testing.T) *config.Profile { t.Helper() liveProfileConfigureMu.Lock() diff --git a/e2e/testdata/fake-drive9.go b/e2e/testdata/fake-drive9.go index 8596bfb..991c533 100644 --- a/e2e/testdata/fake-drive9.go +++ b/e2e/testdata/fake-drive9.go @@ -1,9 +1,12 @@ package main import ( + "encoding/base64" "encoding/json" "fmt" "os" + "sort" + "strings" ) type call struct { @@ -19,6 +22,13 @@ type call struct { Drive9Private string `json:"drive9_private_key,omitempty"` } +type tenant struct { + TenantID string `json:"tenant_id"` + Status string `json:"status"` + Kind string `json:"kind"` + RegionCode string `json:"region_code"` +} + func main() { record := os.Getenv("FAKE_DRIVE9_RECORD") if record != "" { @@ -41,18 +51,53 @@ func main() { _ = file.Close() } args := os.Args[1:] - if len(args) >= 1 && args[0] == "create" { - name := flagValue(args, "--name") + if hasPrefix(args, "create") { + region := flagValue(args, "--region-code") + id := "tenant-" + strings.ReplaceAll(region, "_", "-") + state := loadState() + state[id] = tenant{TenantID: id, Status: "active", Kind: "tidb_cloud", RegionCode: region} + saveState(state) _ = json.NewEncoder(os.Stdout).Encode(map[string]string{ - "tenant_id": "tenant-" + name, - "api_key": "key-" + name, + "tenant_id": id, + "api_key": tokenFor(id), "status": "provisioned", "cloud_provider": "aws", - "region_code": os.Getenv("DRIVE9_REGION_CODE"), + "region_code": region, }) return } - if len(args) >= 1 && args[0] == "delete" { + if hasPrefix(args, "admin", "tenant", "list") { + state := loadState() + tenants := make([]tenant, 0, len(state)) + for _, item := range state { + if item.RegionCode == os.Getenv("DRIVE9_REGION_CODE") { + tenants = append(tenants, item) + } + } + sort.Slice(tenants, func(i, j int) bool { return tenants[i].TenantID < tenants[j].TenantID }) + _ = json.NewEncoder(os.Stdout).Encode(map[string]any{"tenants": tenants, "page": 1, "page_size": 100, "next_page": 0}) + return + } + if hasPrefix(args, "admin", "tenant", "get") { + id := flagValue(args, "--tenant-id") + item, ok := loadState()[id] + if !ok || item.RegionCode != os.Getenv("DRIVE9_REGION_CODE") { + fmt.Fprintln(os.Stderr, "tenant not found") + os.Exit(1) + } + _ = json.NewEncoder(os.Stdout).Encode(item) + return + } + if hasPrefix(args, "admin", "tenant", "delete") { + state := loadState() + id := flagValue(args, "--tenant-id") + item, ok := state[id] + if !ok || item.RegionCode != os.Getenv("DRIVE9_REGION_CODE") { + fmt.Fprintln(os.Stderr, "tenant not found") + os.Exit(1) + } + delete(state, id) + saveState(state) fmt.Println(`{"status":"deleting"}`) return } @@ -65,10 +110,53 @@ func main() { return } if len(args) >= 2 && args[0] == "fs" && args[1] == "stat" { + if expected := os.Getenv("FAKE_DRIVE9_EXPECT_API_KEY"); expected != "" && os.Getenv("DRIVE9_API_KEY") != expected { + fmt.Fprintln(os.Stderr, "fs stat: unauthorized") + os.Exit(1) + } fmt.Println(`{"path":"/","size":0,"isdir":true}`) } } +func hasPrefix(args []string, want ...string) bool { + if len(args) < len(want) { + return false + } + for i := range want { + if args[i] != want[i] { + return false + } + } + return true +} + +func loadState() map[string]tenant { + state := map[string]tenant{} + data, err := os.ReadFile(os.Getenv("FAKE_DRIVE9_STATE")) + if err == nil { + _ = json.Unmarshal(data, &state) + } + return state +} + +func saveState(state map[string]tenant) { + path := os.Getenv("FAKE_DRIVE9_STATE") + if path == "" { + return + } + data, _ := json.Marshal(state) + if err := os.WriteFile(path, data, 0o600); err != nil { + panic(err) + } +} + +func tokenFor(id string) string { + header := base64.RawURLEncoding.EncodeToString([]byte(`{"alg":"none"}`)) + payload, _ := json.Marshal(map[string]string{"tenant_id": id}) + jwt := header + "." + base64.RawURLEncoding.EncodeToString(payload) + ".signature" + return "drive9_" + base64.RawURLEncoding.EncodeToString([]byte(jwt)) +} + func flagValue(args []string, name string) string { for i := 0; i+1 < len(args); i++ { if args[i] == name { diff --git a/internal/cli/commands.go b/internal/cli/commands.go index f7277e1..c8a97e3 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -2,6 +2,9 @@ package cli import ( "fmt" + "io" + "os" + "runtime" "strings" "time" @@ -38,7 +41,7 @@ func newConfigureCommand(info version.Info) *cobra.Command { return apperr.New("config.empty_profile", "usage", 2, "--profile cannot be empty") } } else { - envProfile, _, _, err := envcompat.ResolveNames(nil, "TI_PROFILE", "TDC_PROFILE") + envProfile, _, _, err := envcompat.ResolveNames(nil, "TI_PROFILE", envcompat.LegacyNameFor("TI_PROFILE")) if err != nil { return err } @@ -868,6 +871,7 @@ func newFSCommand(info version.Info) *cobra.Command { newFSDeleteFileSystemCommand(info), newFSListFileSystemsCommand(info), newFSDescribeFileSystemCommand(info), + newFSImportFileSystemTokenCommand(info), newFSCheckFileSystemCommand(info), newFSCopyFileCommand(info), newFSReadFileCommand(info), @@ -894,11 +898,13 @@ func newFSCommand(info version.Info) *cobra.Command { newFSDrainFileSystemCommand(info), newFSUnmountFileSystemCommand(info), } - addFSSelectorFlags(commands, "create-file-system", "list-file-systems", "drain-file-system", "unmount-file-system") + addFSSelectorFlags(commands, "create-file-system", "list-file-systems", "describe-file-system", "delete-file-system", "import-file-system-token", "drain-file-system", "unmount-file-system") addFSAuthFlags(commands, "create-file-system", "list-file-systems", "describe-file-system", + "delete-file-system", + "import-file-system-token", "drain-file-system", "unmount-file-system", ) @@ -915,8 +921,8 @@ func addFSSelectorFlags(commands []*cobra.Command, excluded ...string) { if _, ok := skip[command.Name()]; ok { continue } - if command.Flags().Lookup("file-system-name") == nil { - command.Flags().String("file-system-name", "", "The name of the file system. Can also be supplied through TI_FS_FILE_SYSTEM_NAME.") + if command.Flags().Lookup("file-system-id") == nil { + command.Flags().String("file-system-id", "", "The file system ID. Can also be supplied through TI_FS_FILE_SYSTEM_ID or derived from an explicitly supplied FS token.") } } } @@ -947,11 +953,7 @@ func newFSCreateFileSystemCommand(info version.Info) *cobra.Command { if err != nil { return nil, err } - if err := fscred.MigrateLegacy(profile.HomeDir, profile); err != nil { - return nil, err - } - name, err := ctx.StringFlag("file-system-name") - if err != nil { + if err := fscred.MigrateNameRegistry(profile.HomeDir, profile); err != nil { return nil, err } waitUntilReady, err := ctx.BoolFlag("wait") @@ -960,7 +962,6 @@ func newFSCreateFileSystemCommand(info version.Info) *cobra.Command { } return service.CreateFileSystem(ctx.cmd.Context(), tifs.CreateFileSystemOptions{ Profile: profile, - FileSystemName: name, WaitUntilReady: waitUntilReady, }) }, @@ -969,35 +970,28 @@ func newFSCreateFileSystemCommand(info version.Info) *cobra.Command { if err != nil { return dryrun.Result{}, err } - name, err := ctx.StringFlag("file-system-name") - if err != nil { - return dryrun.Result{}, err - } waitUntilReady, err := ctx.BoolFlag("wait") if err != nil { return dryrun.Result{}, err } return service.DryRunCreateFileSystem(ctx.cmd.Context(), ctx.CommandPath(), tifs.CreateFileSystemOptions{ Profile: profile, - FileSystemName: name, WaitUntilReady: waitUntilReady, }) }, }, info) - cmd.Flags().String("file-system-name", "", "The name of the file system.") cmd.Flags().Bool("wait", false, "Wait until the created file system is active.") - markUsageRequired(cmd, "file-system-name") return cmd } func newFSListFileSystemsCommand(info version.Info) *cobra.Command { return newControlPlaneCommand(controlPlaneCommandSpec{ Use: "list-file-systems", - Short: "List locally registered file systems. (preview)", + Short: "List remote file systems in the selected region. (preview)", Mutation: readOnlyCommand, Permission: authz.FSVolumeRead, Run: func(ctx commandContext) (any, error) { - service, profile, err := fsLocalServiceAndProfile(ctx) + service, profile, err := fsTIServiceAndProfile(ctx) if err != nil { return nil, err } @@ -1013,15 +1007,19 @@ func newFSDescribeFileSystemCommand(info version.Info) *cobra.Command { Mutation: readOnlyCommand, Permission: authz.FSVolumeRead, Run: func(ctx commandContext) (any, error) { - service, profile, err := fsRegistryServiceAndProfile(ctx) + service, profile, err := fsTIServiceAndProfile(ctx) + if err != nil { + return nil, err + } + fileSystemID, err := ctx.StringFlag("file-system-id") if err != nil { return nil, err } - return service.DescribeFileSystem(ctx.cmd.Context(), profile) + return service.DescribeFileSystem(ctx.cmd.Context(), profile, fileSystemID) }, }, info) - cmd.Flags().String("file-system-name", "", "The name of the file system.") - markUsageRequired(cmd, "file-system-name") + cmd.Flags().String("file-system-id", "", "The file system ID.") + markUsageRequired(cmd, "file-system-id") return cmd } @@ -1032,36 +1030,72 @@ func newFSDeleteFileSystemCommand(info version.Info) *cobra.Command { Mutation: mutatingCommand, Permission: authz.FSVolumeDelete, Run: func(ctx commandContext) (any, error) { - service, profile, err := fsTIResourceServiceAndProfile(ctx) + service, profile, err := fsTIServiceAndProfile(ctx) if err != nil { return nil, err } - name, err := fsDeleteFileSystemName(ctx) + fileSystemID, err := ctx.StringFlag("file-system-id") if err != nil { return nil, err } return service.DeleteFileSystem(ctx.cmd.Context(), tifs.DeleteFileSystemOptions{ - Profile: profile, - FileSystemName: name, + Profile: profile, + FileSystemID: fileSystemID, }) }, DryRun: func(ctx commandContext) (dryrun.Result, error) { - service, profile, err := fsTIResourceServiceAndProfile(ctx) + service, profile, err := fsTIServiceAndProfile(ctx) if err != nil { return dryrun.Result{}, err } - name, err := fsDeleteFileSystemName(ctx) + fileSystemID, err := ctx.StringFlag("file-system-id") if err != nil { return dryrun.Result{}, err } return service.DryRunDeleteFileSystem(ctx.cmd.Context(), ctx.CommandPath(), tifs.DeleteFileSystemOptions{ - Profile: profile, - FileSystemName: name, + Profile: profile, + FileSystemID: fileSystemID, }) }, }, info) - cmd.Flags().String("file-system-name", "", "The name of the file system.") - markUsageRequired(cmd, "file-system-name") + cmd.Flags().String("file-system-id", "", "The file system ID.") + markUsageRequired(cmd, "file-system-id") + return cmd +} + +func newFSImportFileSystemTokenCommand(info version.Info) *cobra.Command { + cmd := newControlPlaneCommand(controlPlaneCommandSpec{ + Use: "import-file-system-token", + Short: "Import an existing file system token into local credentials.", + Mutation: mutatingCommand, + Permission: authz.FSVolumeRead, + Run: func(ctx commandContext) (any, error) { + service, profile, err := fsLocalServiceAndProfile(ctx) + if err != nil { + return nil, err + } + opts, err := fsImportTokenOptions(ctx, profile) + if err != nil { + return nil, err + } + return service.ImportFileSystemToken(ctx.cmd.Context(), opts) + }, + DryRun: func(ctx commandContext) (dryrun.Result, error) { + service, profile, err := fsLocalServiceAndProfile(ctx) + if err != nil { + return dryrun.Result{}, err + } + opts, err := fsImportTokenOptions(ctx, profile) + if err != nil { + return dryrun.Result{}, err + } + return service.DryRunImportFileSystemToken(ctx.cmd.Context(), ctx.CommandPath(), opts) + }, + }, info) + cmd.Flags().String("file-system-id", "", "Optional file system ID assertion; it must match the verified token.") + cmd.Flags().String("fs-token", "", "File system token. Prefer TI_FS_TOKEN or --from-file to avoid exposing it in process arguments.") + cmd.Flags().String("from-file", "", "Read the file system token from an owner-only file, or use - for stdin.") + cmd.Flags().Bool("replace", false, "Replace an existing local token for the same file system after validation.") return cmd } @@ -1826,7 +1860,7 @@ func newFSMountFileSystemCommand(info version.Info) *cobra.Command { return service.DryRunMountFileSystem(ctx.cmd.Context(), ctx.CommandPath(), opts) }, }, info) - cmd.Flags().String("file-system-name", "", "The name of the file system. Can also be supplied through TI_FS_FILE_SYSTEM_NAME.") + cmd.Flags().String("file-system-id", "", "The file system ID. Can also be supplied through TI_FS_FILE_SYSTEM_ID or derived from an explicitly supplied FS token.") cmd.Flags().String("mount-path", "", "Local mount path.") cmd.Flags().String("remote-path", "/", "The TiDB Cloud file system root path to mount.") cmd.Flags().String("driver", "auto", "Mount driver: auto, fuse, or webdav.") @@ -1835,7 +1869,7 @@ func newFSMountFileSystemCommand(info version.Info) *cobra.Command { cmd.Flags().Duration("ready-timeout", 30*time.Second, "Time to wait for a background mount to become ready.") cmd.Flags().String("cache-dir", "", "Local FUSE cache directory. Default: ~/.ti/cache/mounts/.") cmd.Flags().Int64("read-cache-size-mb", 128, "FUSE read cache size in MiB. 0 uses the default.") - cmd.Flags().Int64("read-cache-max-file-mb", 4, "Maximumfile size admitted to the FUSE read cache in MiB. 0 uses the default.") + cmd.Flags().Int64("read-cache-max-file-mb", 4, "Maximum file size admitted to the FUSE read cache in MiB. 0 uses the default.") cmd.Flags().Duration("read-cache-ttl", 30*time.Second, "FUSE read cache Time-to-Live.") cmd.Flags().Bool("write-back-cache", true, "Persist FUSE writes locally before writing them to the file system on flush.") cmd.Flags().String("mount-profile", "", "Mount profile: coding-agent, portable, or none. Default: none.") @@ -2257,7 +2291,7 @@ func fsCreateLayerCheckpointOptions(ctx commandContext, profile *config.Profile) } func fsMountOptions(ctx commandContext, profile *config.Profile) (tifs.MountFileSystemOptions, error) { - fileSystemName, err := ctx.StringFlag("file-system-name") + fileSystemID, err := ctx.StringFlag("file-system-id") if err != nil { return tifs.MountFileSystemOptions{}, err } @@ -2327,7 +2361,7 @@ func fsMountOptions(ctx commandContext, profile *config.Profile) (tifs.MountFile } return tifs.MountFileSystemOptions{ Profile: profile, - FileSystemName: fileSystemName, + FileSystemName: fileSystemID, MountPath: mountPath, RemotePath: remotePath, Driver: driver, @@ -2384,21 +2418,6 @@ func fsServiceAndProfile(ctx commandContext) (tifs.Service, *config.Profile, err return fsAuthenticatedServiceAndProfile(ctx, true) } -func fsTIResourceServiceAndProfile(ctx commandContext) (tifs.Service, *config.Profile, error) { - service, profile, err := fsTIServiceAndProfile(ctx) - if err != nil { - return tifs.Service{}, nil, err - } - selected, err := fsResolveAuthenticatedProfile(ctx, profile, true) - if err != nil { - return tifs.Service{}, nil, err - } - if _, err := fscred.Get(profile.HomeDir, profile.Name, selected.FSResourceName); err != nil { - return tifs.Service{}, nil, err - } - return service, selected, nil -} - func fsAuthenticatedServiceAndProfile(ctx commandContext, tokenRequired bool) (tifs.Service, *config.Profile, error) { service, profile, err := fsLocalServiceAndProfile(ctx) if err != nil { @@ -2411,31 +2430,6 @@ func fsAuthenticatedServiceAndProfile(ctx commandContext, tokenRequired bool) (t return service, selected, nil } -func fsRegistryServiceAndProfile(ctx commandContext) (tifs.Service, *config.Profile, error) { - service, profile, err := fsLocalServiceAndProfile(ctx) - if err != nil { - return tifs.Service{}, nil, err - } - selector := "" - selectorExplicit := false - if ctx.cmd.Flag("file-system-name") != nil { - selector, err = ctx.StringFlag("file-system-name") - if err != nil { - return tifs.Service{}, nil, err - } - selectorExplicit = ctx.FlagChanged("file-system-name") - } - resolve := fscred.Resolve - if dryRun, _ := ctx.BoolFlag("dry-run"); dryRun { - resolve = fscred.ResolveDryRun - } - selected, _, err := resolve(profile.HomeDir, profile, selector, selectorExplicit, nil) - if err != nil { - return tifs.Service{}, nil, err - } - return service, selected, nil -} - func fsAdjunctServiceAndProfile(ctx commandContext) (tifs.Service, *config.Profile, error) { return fsServiceAndProfile(ctx) } @@ -2451,13 +2445,13 @@ func fsVaultServiceAndProfile(ctx commandContext) (tifs.Service, *config.Profile func fsResolveAuthenticatedProfile(ctx commandContext, profile *config.Profile, tokenRequired bool) (*config.Profile, error) { selector := "" selectorExplicit := false - if ctx.cmd.Flag("file-system-name") != nil { + if ctx.cmd.Flag("file-system-id") != nil { var err error - selector, err = ctx.StringFlag("file-system-name") + selector, err = ctx.StringFlag("file-system-id") if err != nil { return nil, err } - selectorExplicit = ctx.FlagChanged("file-system-name") + selectorExplicit = ctx.FlagChanged("file-system-id") } token := "" tokenExplicit := false @@ -2473,21 +2467,26 @@ func fsResolveAuthenticatedProfile(ctx commandContext, profile *config.Profile, if flag := ctx.cmd.Flag("region"); flag != nil && flag.Changed { regionOverride = strings.TrimSpace(flag.Value.String()) } else { - value, _, _, err := envcompat.ResolveNames(nil, "TI_REGION_CODE", "TDC_REGION_CODE") + value, _, _, err := envcompat.ResolveNames(nil, "TI_REGION_CODE", envcompat.LegacyNameFor("TI_REGION_CODE")) if err != nil { return nil, err } regionOverride = strings.TrimSpace(value) } dryRun, _ := ctx.BoolFlag("dry-run") - selected, _, err := fscred.ResolveAuthenticated(profile.HomeDir, profile, fscred.ResolveAuthOptions{ - Selector: selector, - SelectorExplicit: selectorExplicit, - Token: token, - TokenExplicit: tokenExplicit, - RegionOverride: regionOverride, - TokenRequired: tokenRequired, - DryRun: dryRun, + if !dryRun { + if err := fscred.MigrateNameRegistry(profile.HomeDir, profile); err != nil { + return nil, err + } + } + selected, _, err := fscred.ResolveCredential(profile.HomeDir, profile, fscred.ResolveCredentialOptions{ + FileSystemID: selector, + FileSystemIDExplicit: selectorExplicit, + Token: token, + TokenExplicit: tokenExplicit, + RegionOverride: regionOverride, + TokenRequired: tokenRequired, + DryRun: dryRun, }) if err != nil { return nil, err @@ -2495,12 +2494,82 @@ func fsResolveAuthenticatedProfile(ctx commandContext, profile *config.Profile, return selected, nil } -func fsDeleteFileSystemName(ctx commandContext) (string, error) { - name, err := ctx.StringFlag("file-system-name") +func fsImportTokenOptions(ctx commandContext, profile *config.Profile) (tifs.ImportFileSystemTokenOptions, error) { + fileSystemID, err := ctx.StringFlag("file-system-id") if err != nil { - return "", err + return tifs.ImportFileSystemTokenOptions{}, err + } + flagToken, err := ctx.StringFlag("fs-token") + if err != nil { + return tifs.ImportFileSystemTokenOptions{}, err + } + fromFile, err := ctx.StringFlag("from-file") + if err != nil { + return tifs.ImportFileSystemTokenOptions{}, err + } + envTokenValue, _, _, err := envcompat.ResolveNames(nil, "TI_FS_TOKEN", envcompat.LegacyNameFor("TI_FS_TOKEN")) + if err != nil { + return tifs.ImportFileSystemTokenOptions{}, err + } + envToken := strings.TrimSpace(envTokenValue) + sources := 0 + if ctx.FlagChanged("fs-token") { + sources++ + } + if strings.TrimSpace(fromFile) != "" { + sources++ + } + if envToken != "" { + sources++ + } + if sources == 0 { + return tifs.ImportFileSystemTokenOptions{}, apperr.New("fs.missing_token", "authentication", 3, "authentication required: pass --fs-token, set TI_FS_TOKEN, or use --from-file") + } + if sources > 1 { + return tifs.ImportFileSystemTokenOptions{}, apperr.New("fs.multiple_token_sources", "usage", 2, "provide exactly one of --fs-token, TI_FS_TOKEN, or --from-file") + } + token := strings.TrimSpace(flagToken) + if strings.TrimSpace(fromFile) != "" { + token, err = readFSImportToken(ctx, fromFile) + if err != nil { + return tifs.ImportFileSystemTokenOptions{}, err + } + } else if token == "" { + token = envToken + } + replace, err := ctx.BoolFlag("replace") + if err != nil { + return tifs.ImportFileSystemTokenOptions{}, err + } + return tifs.ImportFileSystemTokenOptions{Profile: profile, FileSystemID: fileSystemID, Token: token, Replace: replace}, nil +} + +func readFSImportToken(ctx commandContext, path string) (string, error) { + var data []byte + var err error + if path == "-" { + data, err = io.ReadAll(io.LimitReader(ctx.cmd.InOrStdin(), 1<<20)) + } else { + info, statErr := os.Stat(path) + if statErr != nil { + return "", apperr.Wrap("fs.token_file", "usage", 2, "cannot inspect FS token file", statErr) + } + if !info.Mode().IsRegular() { + return "", apperr.New("fs.token_file", "usage", 2, "FS token file must be a regular file") + } + if runtime.GOOS != "windows" && info.Mode().Perm()&0o077 != 0 { + return "", apperr.New("fs.token_file_permissions", "usage", 2, fmt.Sprintf("FS token file %s must have mode 0600 or stricter", path)) + } + data, err = os.ReadFile(path) + } + if err != nil { + return "", apperr.Wrap("fs.token_file", "usage", 2, "cannot read FS token", err) + } + token := strings.TrimSpace(string(data)) + if token == "" { + return "", apperr.New("fs.empty_token", "usage", 2, "FS token input is empty") } - return name, nil + return token, nil } func newFSVaultCommand(info version.Info) *cobra.Command { @@ -3006,7 +3075,7 @@ func vaultToken(ctx commandContext) (string, error) { if token != "" { return token, nil } - value, _, _, err := envcompat.ResolveNames(nil, "TI_VAULT_TOKEN", "TDC_VAULT_TOKEN") + value, _, _, err := envcompat.ResolveNames(nil, "TI_VAULT_TOKEN", envcompat.LegacyNameFor("TI_VAULT_TOKEN")) return value, err } diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index 3799e03..9b35131 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -830,7 +830,7 @@ func TestFSAdjunctCommandsRequireConfiguredFSResource(t *testing.T) { if got := apperr.ExitCodeFor(err); got != 2 { t.Fatalf("expected config exit code 2, got %d", got) } - if got := apperr.MessageFor(err); !strings.Contains(got, "file system name is required") || !strings.Contains(got, "TI_FS_FILE_SYSTEM_NAME") { + if got := apperr.MessageFor(err); !strings.Contains(got, "file system ID is required") || !strings.Contains(got, "TI_FS_FILE_SYSTEM_ID") { t.Fatalf("unexpected message %q", got) } } @@ -838,6 +838,7 @@ func TestFSAdjunctCommandsRequireConfiguredFSResource(t *testing.T) { func TestFSOperationalCommandsExposeResourceSelector(t *testing.T) { root := NewRootCommand(testVersion()) excluded := map[string]bool{ + "ti fs create-file-system": true, "ti fs list-file-systems": true, "ti fs drain-file-system": true, "ti fs unmount-file-system": true, @@ -851,8 +852,11 @@ func TestFSOperationalCommandsExposeResourceSelector(t *testing.T) { if !strings.HasPrefix(path, "ti fs ") && !strings.HasPrefix(path, "ti fs-git ") && !strings.HasPrefix(path, "ti fs-journal ") && !strings.HasPrefix(path, "ti fs-vault ") { return } - if cmd.Flags().Lookup("file-system-name") == nil { - t.Fatalf("%s does not expose --file-system-name", path) + if cmd.Flags().Lookup("file-system-id") == nil { + t.Fatalf("%s does not expose --file-system-id", path) + } + if cmd.Flags().Lookup("file-system-name") != nil { + t.Fatalf("%s still exposes removed --file-system-name", path) } }) } @@ -863,6 +867,7 @@ func TestFSRemoteCommandsExposeTokenFlag(t *testing.T) { "ti fs create-file-system": true, "ti fs list-file-systems": true, "ti fs describe-file-system": true, + "ti fs delete-file-system": true, "ti fs drain-file-system": true, "ti fs unmount-file-system": true, "ti fs-vault unmount-vault": true, @@ -896,7 +901,7 @@ func TestFSRegistryDryRunDoesNotMigrateLegacyState(t *testing.T) { }, store.CredentialsProfile{TiDBCloudPublicKey: "public", TiDBCloudPrivateKey: "private", FSAPIKey: "key-1"}); err != nil { t.Fatal(err) } - _, _, err := executeForTest("fs", "copy-file", "--file-system-name", "workspace", "--from-remote", "/source", "--to-remote", "/target", "--dry-run") + _, _, err := executeForTest("fs", "copy-file", "--file-system-id", "tenant-1", "--from-remote", "/source", "--to-remote", "/target", "--dry-run") if err != nil { t.Fatalf("dry-run failed: %v", err) } diff --git a/internal/config/envcompat/envcompat.go b/internal/config/envcompat/envcompat.go index e2b0106..6418b98 100644 --- a/internal/config/envcompat/envcompat.go +++ b/internal/config/envcompat/envcompat.go @@ -20,7 +20,7 @@ var PublicPairs = []Pair{ {Canonical: "TIDB_CLOUD_PUBLIC_KEY", Legacy: "TDC_PUBLIC_KEY"}, {Canonical: "TIDB_CLOUD_PRIVATE_KEY", Legacy: "TDC_PRIVATE_KEY"}, {Canonical: "TI_FS_TOKEN", Legacy: "TDC_FS_TOKEN"}, - {Canonical: "TI_FS_FILE_SYSTEM_NAME", Legacy: "TDC_FS_FILE_SYSTEM_NAME"}, + {Canonical: "TI_FS_FILE_SYSTEM_ID", Legacy: "TDC_FS_FILE_SYSTEM_ID"}, {Canonical: "TI_LOGGING", Legacy: "TDC_LOGGING"}, {Canonical: "TI_TELEMETRY", Legacy: "TDC_TELEMETRY"}, {Canonical: "TI_TELEMETRY_TAG", Legacy: "TDC_TELEMETRY_TAG"}, diff --git a/internal/fs/control.go b/internal/fs/control.go index fb4dfd1..f195073 100644 --- a/internal/fs/control.go +++ b/internal/fs/control.go @@ -40,29 +40,45 @@ type Service struct { type CreateFileSystemOptions struct { Profile *config.Profile - FileSystemName string WaitUntilReady bool } type DeleteFileSystemOptions struct { - Profile *config.Profile - FileSystemName string + Profile *config.Profile + FileSystemID string } type CheckFileSystemOptions struct { Profile *config.Profile } +type ImportFileSystemTokenOptions struct { + Profile *config.Profile + FileSystemID string + Token string + Replace bool +} + +type FileSystemSummary struct { + FileSystemID string `json:"file_system_id"` + RegionCode string `json:"region_code,omitempty"` + Status string `json:"status,omitempty"` + Kind string `json:"kind,omitempty"` + Quota any `json:"quota,omitempty"` + HasLocalToken bool `json:"has_local_token"` +} + +type ListFileSystemsResult struct { + RegionCode string `json:"region_code"` + FileSystems []FileSystemSummary `json:"file_systems"` +} + type DescribeFileSystemResult struct { - Profile string `json:"profile"` - fscred.Resource - Drive9Home string `json:"drive9_home"` + FileSystemSummary } type FileSystemResult struct { - FileSystemName string `json:"file_system_name"` - TenantID string `json:"tenant_id,omitempty"` - CloudProvider string `json:"cloud_provider,omitempty"` + FileSystemID string `json:"file_system_id"` RegionCode string `json:"region_code,omitempty"` FSToken string `json:"fs_token,omitempty"` Status string `json:"status"` @@ -70,17 +86,23 @@ type FileSystemResult struct { } type DeleteResult struct { - FileSystemName string `json:"file_system_name"` - TenantID string `json:"tenant_id,omitempty"` + FileSystemID string `json:"file_system_id"` Status string `json:"status"` CredentialsRemoved bool `json:"credentials_removed"` RemoteDeletionState string `json:"remote_deletion_state,omitempty"` } +type ImportFileSystemTokenResult struct { + FileSystemID string `json:"file_system_id"` + RegionCode string `json:"region_code"` + CredentialsStored bool `json:"credentials_stored"` + Status string `json:"status"` +} + type CheckResult struct { Status string `json:"status"` Profile string `json:"profile"` - Resource fscred.Resource `json:"resource"` + Resource fscred.Credential `json:"resource"` Endpoint *endpoints.Endpoint `json:"endpoint,omitempty"` Remote *apifs.StatusResponse `json:"remote,omitempty"` Checks []Check `json:"checks"` @@ -104,43 +126,45 @@ func (s Service) CheckFileSystem(ctx context.Context, opts CheckFileSystemOption return s.drive9CheckFileSystem(ctx, opts) } -func (s Service) ListFileSystems(_ context.Context, profile *config.Profile) (fscred.ListResult, error) { - homeDir, err := s.homeDir() - if err != nil { - return fscred.ListResult{}, err - } - if err := fscred.MigrateLegacy(homeDir, profile); err != nil { - return fscred.ListResult{}, err - } - resources, err := fscred.List(homeDir, profileName(profile)) - if err != nil { - return fscred.ListResult{}, err - } - return fscred.ListResult{Profile: profileName(profile), FileSystems: resources}, nil +func (s Service) ListFileSystems(ctx context.Context, profile *config.Profile) (ListFileSystemsResult, error) { + return s.drive9ListFileSystems(ctx, profile) } -func (s Service) DescribeFileSystem(_ context.Context, profile *config.Profile) (DescribeFileSystemResult, error) { - resource := fscred.FromProfile(profile) - homeDir, err := s.homeDir() - if err != nil { - return DescribeFileSystemResult{}, err - } - drive9Home, err := fscred.CompanionHome(homeDir, profileName(profile), resource.Name) +func (s Service) DescribeFileSystem(ctx context.Context, profile *config.Profile, fileSystemID string) (DescribeFileSystemResult, error) { + return s.drive9DescribeFileSystem(ctx, profile, fileSystemID) +} + +func (s Service) ImportFileSystemToken(ctx context.Context, opts ImportFileSystemTokenOptions) (ImportFileSystemTokenResult, error) { + return s.importFileSystemToken(ctx, opts, true) +} + +func (s Service) DryRunImportFileSystemToken(ctx context.Context, commandPath string, opts ImportFileSystemTokenOptions) (dryrun.Result, error) { + result, err := s.importFileSystemToken(ctx, opts, false) if err != nil { - return DescribeFileSystemResult{}, err + return dryrun.Result{}, err } - return DescribeFileSystemResult{Profile: profileName(profile), Resource: resource, Drive9Home: drive9Home}, nil + return dryrun.New( + commandPath, + "import_file_system_token", + dryrun.RequestSummary{ + Method: "EXEC", + Path: "ti-drive9 fs stat --output json :/", + Description: "the companion verifies access to the remote root; normal execution then stores the token in the selected local profile namespace", + }, + dryrun.Check{Name: "token_validation", Status: "passed", Message: result.FileSystemID}, + dryrun.Check{Name: "credential_destination", Status: "passed", Message: profileName(opts.Profile)}, + ), nil } func (s Service) DryRunCreateFileSystem(ctx context.Context, commandPath string, opts CreateFileSystemOptions) (dryrun.Result, error) { - request, name, endpoint, endpointErr, err := s.createDryRunInputs(opts) + request, endpoint, endpointErr, err := s.createDryRunInputs(opts) if err != nil { return dryrun.Result{}, err } checks := []dryrun.Check{ {Name: "config_and_credentials", Status: "passed", Message: fmt.Sprintf("profile %q loaded", profileName(opts.Profile))}, {Name: "permission_requirement", Status: "passed", Message: string(authz.FSVolumeCreate)}, - {Name: "file_system_name", Status: "passed", Message: name}, + {Name: "remote_identity", Status: "passed", Message: "Drive9 assigns file_system_id"}, } checks = append(checks, endpointDryRunCheck(endpoint, endpointErr)) if opts.WaitUntilReady { @@ -163,7 +187,7 @@ func (s Service) DryRunCreateFileSystem(ctx context.Context, commandPath string, } func (s Service) DryRunDeleteFileSystem(ctx context.Context, commandPath string, opts DeleteFileSystemOptions) (dryrun.Result, error) { - name, endpoint, endpointErr, err := s.deleteDryRunInputs(opts) + fileSystemID, endpoint, endpointErr, err := s.deleteDryRunInputs(opts) if err != nil { return dryrun.Result{}, err } @@ -171,23 +195,19 @@ func (s Service) DryRunDeleteFileSystem(ctx context.Context, commandPath string, {Name: "config_and_credentials", Status: "passed", Message: fmt.Sprintf("profile %q loaded", profileName(opts.Profile))}, {Name: "permission_requirement", Status: "passed", Message: string(authz.FSVolumeDelete)}, } - if resource := fscred.FromProfile(opts.Profile); resource.HasAPIKey { - checks = append(checks, dryrun.Check{Name: "fs_resource_credentials", Status: "passed", Message: name}) - } else { - checks = append(checks, dryrun.Check{Name: "fs_resource_credentials", Status: "warning", Message: "fs_api_key is not configured; normal execution would fail before remote deletion"}) - } + checks = append(checks, dryrun.Check{Name: "file_system_id", Status: "passed", Message: fileSystemID}) homeDir, err := s.homeDir() if err != nil { return dryrun.Result{}, err } - registryPaths, err := fscred.Paths(homeDir, profileName(opts.Profile), name) + credentialPaths, err := fscred.CredentialPath(homeDir, profileName(opts.Profile), fileSystemID) if err != nil { return dryrun.Result{}, err } checks = append(checks, dryrun.Check{ - Name: "local_resource_registry", + Name: "local_credentials", Status: "passed", - Message: fmt.Sprintf("would remove %s and %s", registryPaths.Config, registryPaths.Credentials), + Message: fmt.Sprintf("would remove %s after Drive9 accepts deletion if it exists", credentialPaths.Credentials), }) checks = append(checks, endpointDryRunCheck(endpoint, endpointErr)) body, bodyErr := deprovisionRequest(opts.Profile) @@ -199,66 +219,59 @@ func (s Service) DryRunDeleteFileSystem(ctx context.Context, commandPath string, "delete_file_system", dryrun.RequestSummary{ Method: http.MethodDelete, - Path: "/v1/tenant", + Path: "/v1/admin/tenants/" + fileSystemID, Body: redactedDeprovisionRequest(body), - Description: "normal execution uses the stored ti fs API key before deleting", + Description: "normal execution uses TiDB Cloud credentials and removes matching local credentials only after Drive9 accepts deletion", }, checks..., ), nil } -func (s Service) createRequestAndEndpoint(opts CreateFileSystemOptions, requireEndpoint bool) (apifs.ProvisionRequest, string, endpoints.Endpoint, error) { - request, name, endpoint, endpointErr, err := s.createDryRunInputs(opts) +func (s Service) createRequestAndEndpoint(opts CreateFileSystemOptions, requireEndpoint bool) (apifs.ProvisionRequest, endpoints.Endpoint, error) { + request, endpoint, endpointErr, err := s.createDryRunInputs(opts) if err != nil { - return apifs.ProvisionRequest{}, "", endpoints.Endpoint{}, err + return apifs.ProvisionRequest{}, endpoints.Endpoint{}, err } if endpointErr != nil && requireEndpoint { - return apifs.ProvisionRequest{}, "", endpoints.Endpoint{}, endpointErr + return apifs.ProvisionRequest{}, endpoints.Endpoint{}, endpointErr } - return request, name, endpoint, nil + return request, endpoint, nil } -func (s Service) createDryRunInputs(opts CreateFileSystemOptions) (apifs.ProvisionRequest, string, endpoints.Endpoint, error, error) { +func (s Service) createDryRunInputs(opts CreateFileSystemOptions) (apifs.ProvisionRequest, endpoints.Endpoint, error, error) { creds, err := auth.ValidateProfile(opts.Profile) if err != nil { - return apifs.ProvisionRequest{}, "", endpoints.Endpoint{}, nil, err - } - name, err := fileSystemName(opts.FileSystemName) - if err != nil { - return apifs.ProvisionRequest{}, "", endpoints.Endpoint{}, nil, err + return apifs.ProvisionRequest{}, endpoints.Endpoint{}, nil, err } endpoint, endpointErr := s.resolveFS(opts.Profile) request := apifs.ProvisionRequest{ PublicKey: creds.PublicKey, PrivateKey: creds.PrivateKey, } - return request, name, endpoint, endpointErr, nil + return request, endpoint, endpointErr, nil } func (s Service) deleteInputsAndEndpoint(opts DeleteFileSystemOptions, requireEndpoint bool) (string, endpoints.Endpoint, error) { - name, endpoint, endpointErr, err := s.deleteDryRunInputs(opts) + fileSystemID, endpoint, endpointErr, err := s.deleteDryRunInputs(opts) if err != nil { return "", endpoints.Endpoint{}, err } if endpointErr != nil && requireEndpoint { return "", endpoints.Endpoint{}, endpointErr } - return name, endpoint, nil + return fileSystemID, endpoint, nil } func (s Service) deleteDryRunInputs(opts DeleteFileSystemOptions) (string, endpoints.Endpoint, error, error) { if err := validateProfile(opts.Profile); err != nil { return "", endpoints.Endpoint{}, nil, err } - name, err := fileSystemName(opts.FileSystemName) + fileSystemID, err := fscred.ValidateFileSystemID(opts.FileSystemID) if err != nil { return "", endpoints.Endpoint{}, nil, err } - if opts.Profile.FSResourceName != name { - return "", endpoints.Endpoint{}, nil, resourceMismatch(opts.Profile.FSResourceName, name) - } endpoint, endpointErr := s.resolveFS(opts.Profile) - return name, endpoint, endpointErr, nil + return fileSystemID, endpoint, endpointErr, nil } func (s Service) resolveFS(profile *config.Profile) (endpoints.Endpoint, error) { @@ -374,31 +387,6 @@ func validateProfile(profile *config.Profile) error { return nil } -func fileSystemName(value string) (string, error) { - trimmed := strings.TrimSpace(value) - if trimmed == "" { - return "", apperr.New("fs.missing_file_system_name", "usage", 2, "--file-system-name is required") - } - if len(trimmed) > 64 || strings.Contains(trimmed, "/") { - return "", apperr.New("fs.invalid_file_system_name", "usage", 2, "--file-system-name must be 1-64 characters and must not contain /") - } - for _, r := range trimmed { - if r < 0x20 || r == 0x7f { - return "", apperr.New("fs.invalid_file_system_name", "usage", 2, "--file-system-name must not contain control characters") - } - } - return trimmed, nil -} - -func resourceMismatch(existing, requested string) error { - return apperr.New( - "fs.resource_name_mismatch", - "usage", - 2, - fmt.Sprintf("profile is already configured for ti fs resource %q; use that name or delete it before creating %q", existing, requested), - ) -} - func endpointDryRunCheck(endpoint endpoints.Endpoint, err error) dryrun.Check { if err != nil { return dryrun.Check{Name: "endpoint_selection", Status: "skipped", Message: apperr.MessageFor(err)} @@ -406,7 +394,7 @@ func endpointDryRunCheck(endpoint endpoints.Endpoint, err error) dryrun.Check { return dryrun.Check{Name: "endpoint_selection", Status: "passed", Message: fmt.Sprintf("%s %s", endpoint.Provider, endpoint.RegionCode)} } -func checkResult(profile *config.Profile, resource fscred.Resource, endpoint *endpoints.Endpoint, remote *apifs.StatusResponse, checks []Check) CheckResult { +func checkResult(profile *config.Profile, resource fscred.Credential, endpoint *endpoints.Endpoint, remote *apifs.StatusResponse, checks []Check) CheckResult { status := "passed" for _, check := range checks { if check.Status == "failed" { @@ -436,38 +424,41 @@ func profileName(profile *config.Profile) string { func (r FileSystemResult) Human() string { lines := []string{ - "File system: " + r.FileSystemName, + "File system ID: " + r.FileSystemID, "Status: " + r.Status, } - if r.TenantID != "" { - lines = append(lines, "Tenant ID: "+r.TenantID) - } - if r.CloudProvider != "" || r.RegionCode != "" { - lines = append(lines, "Location: "+strings.TrimSpace(r.CloudProvider+" "+r.RegionCode)) + if r.RegionCode != "" { + lines = append(lines, "Region: "+r.RegionCode) } if r.CredentialsStored { - lines = append(lines, "Credentials: stored in ~/.ti/credentials") + lines = append(lines, "Credentials: stored locally") } return strings.Join(lines, "\n") } func (r DeleteResult) Human() string { lines := []string{ - "File system: " + r.FileSystemName, + "File system ID: " + r.FileSystemID, "Status: " + r.Status, } - if r.TenantID != "" { - lines = append(lines, "Tenant ID: "+r.TenantID) - } if r.RemoteDeletionState != "" { lines = append(lines, "Remote deletion state: "+r.RemoteDeletionState) } if r.CredentialsRemoved { - lines = append(lines, "Credentials: removed from ~/.ti/credentials") + lines = append(lines, "Credentials: removed from ~/.ti/fs_credentials") } return strings.Join(lines, "\n") } +func (r ImportFileSystemTokenResult) Human() string { + return strings.Join([]string{ + "File system ID: " + r.FileSystemID, + "Region: " + r.RegionCode, + "Status: " + r.Status, + "Credentials: stored locally", + }, "\n") +} + func (r CheckResult) Human() string { var out strings.Builder _, _ = fmt.Fprintf(&out, "ti fs check: %s\n", r.Status) diff --git a/internal/fs/drive9_companion.go b/internal/fs/drive9_companion.go index bc79c8e..89751d0 100644 --- a/internal/fs/drive9_companion.go +++ b/internal/fs/drive9_companion.go @@ -42,8 +42,23 @@ type drive9CreateOutput struct { } type drive9DeleteOutput struct { - Status string `json:"status"` - Server string `json:"server,omitempty"` + TenantID string `json:"tenant_id,omitempty"` + Status string `json:"status"` + Server string `json:"server,omitempty"` +} + +type drive9AdminTenant struct { + TenantID string `json:"tenant_id"` + Status string `json:"status"` + Kind string `json:"kind"` + Quota any `json:"quota,omitempty"` +} + +type drive9AdminTenantListOutput struct { + Tenants []drive9AdminTenant `json:"tenants"` + Page int `json:"page"` + PageSize int `json:"page_size"` + NextPage int `json:"next_page,omitempty"` } type drive9StatMetadata struct { @@ -151,7 +166,7 @@ func sleepDrive9Retry(ctx context.Context, attempt int) error { } func (s Service) drive9CreateFileSystem(ctx context.Context, opts CreateFileSystemOptions) (FileSystemResult, error) { - _, name, _, err := s.createRequestAndEndpoint(opts, false) + _, _, err := s.createRequestAndEndpoint(opts, false) if err != nil { return FileSystemResult{}, err } @@ -159,30 +174,23 @@ func (s Service) drive9CreateFileSystem(ctx context.Context, opts CreateFileSyst if err != nil { return FileSystemResult{}, err } - if existing, getErr := fscred.Get(homeDir, opts.Profile.Name, name); getErr == nil { - fileSystem := FileSystemResult{ - FileSystemName: existing.Name, - TenantID: existing.TenantID, - CloudProvider: existing.CloudProvider, - RegionCode: existing.RegionCode, - FSToken: existing.APIKey, - Status: "exists", - CredentialsStored: true, - } - if opts.WaitUntilReady { - if err := s.waitUntilFileSystemReady(ctx, homeDir, opts.Profile, name); err != nil { - return FileSystemResult{}, err - } - fileSystem.Status = "ready" - } - return fileSystem, nil - } else if apperr.CodeFor(getErr) != "fs.resource_not_found" { - return FileSystemResult{}, getErr + if err := fscred.MigrateNameRegistry(homeDir, opts.Profile); err != nil { + return FileSystemResult{}, err } - args := []string{"create", "--json", "--name", name, "--region-code", opts.Profile.PlacementRegionCode} - result, err := s.drive9Runner().Run(ctx, fswrap.RunOptions{ + if err := fscred.PrepareCredentialStore(homeDir, opts.Profile.Name); err != nil { + return FileSystemResult{}, err + } + createHome, err := os.MkdirTemp("", "ti-fs-create-*") + if err != nil { + return FileSystemResult{}, apperr.Wrap("fs.companion_home", "runtime", 1, "prepare temporary ti fs create state", err) + } + defer os.RemoveAll(createHome) + runner := s.drive9Runner() + runner.HomeDir = createHome + args := []string{"create", "--json", "--region-code", opts.Profile.PlacementRegionCode} + result, err := runner.Run(ctx, fswrap.RunOptions{ Profile: opts.Profile, - ResourceName: name, + ResourceName: "_create", Args: args, CaptureStdout: true, IncludeTIKeys: true, @@ -197,33 +205,35 @@ func (s Service) drive9CreateFileSystem(ctx context.Context, opts CreateFileSyst } status := strings.TrimSpace(out.Status) if status == "" { - status = "provisioned" + status = "provisioning" } - cloudProvider := out.CloudProvider - if cloudProvider == "" { - cloudProvider = opts.Profile.CloudProvider + fileSystemID, err := fscred.ValidateFileSystemID(out.TenantID) + if err != nil { + return FileSystemResult{}, apperr.Wrap("fs.companion_decode", "runtime", 1, "ti fs create response did not include a valid tenant_id", err) } - regionCode := out.RegionCode - if regionCode == "" { - regionCode = out.Region + if strings.TrimSpace(out.APIKey) == "" { + return FileSystemResult{}, apperr.New("fs.companion_decode", "runtime", 1, "ti fs create response did not include api_key") } + regionCode := opts.Profile.PlacementRegionCode if regionCode == "" { - regionCode = opts.Profile.PlacementRegionCode - } - if err := fscred.Store(homeDir, opts.Profile, name, out.TenantID, cloudProvider, regionCode, out.APIKey); err != nil { - return FileSystemResult{}, err + regionCode = out.RegionCode } fileSystem := FileSystemResult{ - FileSystemName: name, - TenantID: out.TenantID, - CloudProvider: cloudProvider, + FileSystemID: fileSystemID, RegionCode: regionCode, FSToken: out.APIKey, Status: status, - CredentialsStored: true, + CredentialsStored: false, } + if _, storeErr := fscred.StoreCredential(homeDir, opts.Profile, fileSystemID, regionCode, out.APIKey, false); storeErr != nil { + if s.Stderr != nil { + _, _ = fmt.Fprintf(s.Stderr, "ti [WARNING]: file system %s was created, but its one-time token could not be stored locally: %s\n", fileSystemID, apperr.MessageFor(storeErr)) + } + return fileSystem, nil + } + fileSystem.CredentialsStored = true if opts.WaitUntilReady { - if err := s.waitUntilFileSystemReady(ctx, homeDir, opts.Profile, name); err != nil { + if err := s.waitUntilFileSystemReady(ctx, homeDir, opts.Profile, fileSystemID); err != nil { return FileSystemResult{}, err } fileSystem.Status = "ready" @@ -232,51 +242,223 @@ func (s Service) drive9CreateFileSystem(ctx context.Context, opts CreateFileSyst } func (s Service) drive9DeleteFileSystem(ctx context.Context, opts DeleteFileSystemOptions) (DeleteResult, error) { - name, _, err := s.deleteInputsAndEndpoint(opts, false) + fileSystemID, _, err := s.deleteInputsAndEndpoint(opts, false) + if err != nil { + return DeleteResult{}, err + } + homeDir, err := s.homeDir() if err != nil { return DeleteResult{}, err } - resource := fscred.FromProfile(opts.Profile) - args := []string{"delete", "--json", "--yes"} + if err := fscred.MigrateNameRegistry(homeDir, opts.Profile); err != nil { + return DeleteResult{}, err + } + args := []string{"admin", "tenant", "delete", "--json", "--region-code", opts.Profile.PlacementRegionCode, "--tenant-id", fileSystemID} result, err := s.drive9Runner().Run(ctx, fswrap.RunOptions{ Profile: opts.Profile, + ResourceName: "_control-plane", Args: args, CaptureStdout: true, IncludeTIKeys: true, - IncludeFSAPIKey: true, + IncludeFSAPIKey: false, }) if err != nil { + if isDrive9NotFound(err) { + return DeleteResult{}, remoteFileSystemNotFound(fileSystemID, err) + } return DeleteResult{}, err } var out drive9DeleteOutput if err := json.Unmarshal(result.Stdout, &out); err != nil { return DeleteResult{}, apperr.Wrap("fs.companion_decode", "runtime", 1, "decode ti fs deletion response", err) } + if out.TenantID != "" && out.TenantID != fileSystemID { + return DeleteResult{}, apperr.New("fs.companion_decode", "runtime", 1, fmt.Sprintf("ti fs deletion response identified file system %q instead of %q", out.TenantID, fileSystemID)) + } status := strings.TrimSpace(out.Status) if status == "" { status = "deleting" } - homeDir, err := s.homeDir() + credentialsRemoved, err := fscred.DeleteCredential(homeDir, opts.Profile.Name, fileSystemID) if err != nil { return DeleteResult{}, err } - if err := fscred.Delete(homeDir, opts.Profile, name); err != nil { - return DeleteResult{}, err - } - if companionHome, companionErr := fscred.CompanionHome(homeDir, opts.Profile.Name, name); companionErr == nil { - _ = os.RemoveAll(companionHome) - } return DeleteResult{ - FileSystemName: name, - TenantID: resource.TenantID, + FileSystemID: fileSystemID, Status: status, - CredentialsRemoved: true, + CredentialsRemoved: credentialsRemoved, RemoteDeletionState: status, }, nil } -func (s Service) waitUntilFileSystemReady(ctx context.Context, homeDir string, profile *config.Profile, name string) error { - selected, _, err := fscred.Resolve(homeDir, profile, name, true, nil) +func (s Service) drive9ListFileSystems(ctx context.Context, profile *config.Profile) (ListFileSystemsResult, error) { + if err := validateProfile(profile); err != nil { + return ListFileSystemsResult{}, err + } + homeDir, err := s.homeDir() + if err != nil { + return ListFileSystemsResult{}, err + } + if err := fscred.MigrateNameRegistry(homeDir, profile); err != nil { + return ListFileSystemsResult{}, err + } + credentials, err := fscred.ListCredentials(homeDir, profile.Name) + if err != nil { + return ListFileSystemsResult{}, err + } + hasToken := make(map[string]bool, len(credentials)) + for _, credential := range credentials { + hasToken[credential.FileSystemID] = credential.HasLocalToken + } + const pageSize = 100 + page := 1 + seenPages := map[int]bool{} + seenIDs := map[string]bool{} + fileSystems := make([]FileSystemSummary, 0) + for { + if page <= 0 || seenPages[page] { + return ListFileSystemsResult{}, apperr.New("fs.companion_decode", "runtime", 1, "ti fs inventory returned a repeated or invalid page") + } + seenPages[page] = true + args := []string{"admin", "tenant", "list", "--json", "--region-code", profile.PlacementRegionCode, "--page-size", strconv.Itoa(pageSize), "--page", strconv.Itoa(page)} + result, err := s.drive9Runner().Run(ctx, fswrap.RunOptions{Profile: profile, ResourceName: "_control-plane", Args: args, CaptureStdout: true, IncludeTIKeys: true}) + if err != nil { + return ListFileSystemsResult{}, err + } + var out drive9AdminTenantListOutput + if err := json.Unmarshal(result.Stdout, &out); err != nil { + return ListFileSystemsResult{}, apperr.Wrap("fs.companion_decode", "runtime", 1, "decode ti fs inventory response", err) + } + if out.Page != page { + return ListFileSystemsResult{}, apperr.New("fs.companion_decode", "runtime", 1, fmt.Sprintf("ti fs inventory returned page %d while page %d was requested", out.Page, page)) + } + for _, tenant := range out.Tenants { + id, err := fscred.ValidateFileSystemID(tenant.TenantID) + if err != nil { + return ListFileSystemsResult{}, apperr.Wrap("fs.companion_decode", "runtime", 1, "ti fs inventory included an invalid tenant_id", err) + } + if seenIDs[id] { + return ListFileSystemsResult{}, apperr.New("fs.companion_decode", "runtime", 1, fmt.Sprintf("ti fs inventory returned duplicate file system ID %q", id)) + } + seenIDs[id] = true + fileSystems = append(fileSystems, FileSystemSummary{FileSystemID: id, RegionCode: profile.PlacementRegionCode, Status: tenant.Status, Kind: tenant.Kind, Quota: tenant.Quota, HasLocalToken: hasToken[id]}) + } + if out.NextPage == 0 { + break + } + if out.NextPage <= page { + return ListFileSystemsResult{}, apperr.New("fs.companion_decode", "runtime", 1, "ti fs inventory returned a repeated or regressing next_page") + } + page = out.NextPage + } + sort.Slice(fileSystems, func(i, j int) bool { return fileSystems[i].FileSystemID < fileSystems[j].FileSystemID }) + return ListFileSystemsResult{RegionCode: profile.PlacementRegionCode, FileSystems: fileSystems}, nil +} + +func (s Service) drive9DescribeFileSystem(ctx context.Context, profile *config.Profile, fileSystemID string) (DescribeFileSystemResult, error) { + if err := validateProfile(profile); err != nil { + return DescribeFileSystemResult{}, err + } + id, err := fscred.ValidateFileSystemID(fileSystemID) + if err != nil { + return DescribeFileSystemResult{}, err + } + homeDir, err := s.homeDir() + if err != nil { + return DescribeFileSystemResult{}, err + } + if err := fscred.MigrateNameRegistry(homeDir, profile); err != nil { + return DescribeFileSystemResult{}, err + } + result, err := s.drive9Runner().Run(ctx, fswrap.RunOptions{ + Profile: profile, ResourceName: "_control-plane", + Args: []string{"admin", "tenant", "get", "--json", "--region-code", profile.PlacementRegionCode, "--tenant-id", id}, + CaptureStdout: true, IncludeTIKeys: true, + }) + if err != nil { + if isDrive9NotFound(err) { + return DescribeFileSystemResult{}, remoteFileSystemNotFound(id, err) + } + return DescribeFileSystemResult{}, err + } + var tenant drive9AdminTenant + if err := json.Unmarshal(result.Stdout, &tenant); err != nil { + return DescribeFileSystemResult{}, apperr.Wrap("fs.companion_decode", "runtime", 1, "decode ti fs describe response", err) + } + if tenant.TenantID != id { + return DescribeFileSystemResult{}, apperr.New("fs.companion_decode", "runtime", 1, fmt.Sprintf("ti fs describe response identified file system %q instead of %q", tenant.TenantID, id)) + } + _, credentialErr := fscred.GetCredential(homeDir, profile.Name, id) + if credentialErr != nil && apperr.CodeFor(credentialErr) != "fs.credential_not_found" { + return DescribeFileSystemResult{}, credentialErr + } + return DescribeFileSystemResult{FileSystemSummary: FileSystemSummary{ + FileSystemID: id, RegionCode: profile.PlacementRegionCode, Status: tenant.Status, Kind: tenant.Kind, Quota: tenant.Quota, HasLocalToken: credentialErr == nil, + }}, nil +} + +func (s Service) importFileSystemToken(ctx context.Context, opts ImportFileSystemTokenOptions, persist bool) (ImportFileSystemTokenResult, error) { + if opts.Profile == nil { + return ImportFileSystemTokenResult{}, apperr.New("fs.missing_profile", "config", 2, "active profile is required") + } + tokenID, err := fscred.FileSystemIDFromToken(opts.Token) + if err != nil { + return ImportFileSystemTokenResult{}, err + } + if asserted := strings.TrimSpace(opts.FileSystemID); asserted != "" { + asserted, err = fscred.ValidateFileSystemID(asserted) + if err != nil { + return ImportFileSystemTokenResult{}, err + } + if asserted != tokenID { + return ImportFileSystemTokenResult{}, apperr.New("fs.token_file_system_mismatch", "authentication", 3, fmt.Sprintf("FS token belongs to file system %q, not %q", tokenID, asserted)) + } + } + placement, err := region.ParsePlacementCode(opts.Profile.PlacementRegionCode) + if err != nil { + return ImportFileSystemTokenResult{}, err + } + selected := *opts.Profile + selected.FSResourceName = tokenID + selected.FSTenantID = tokenID + selected.FSPlacementRegionCode = placement.Code + selected.FSCloudProvider = placement.Provider + selected.FSRegionCode = placement.NativeCode + selected.FSAPIKey = opts.Token + validationHome, err := os.MkdirTemp("", "ti-fs-token-validation-*") + if err != nil { + return ImportFileSystemTokenResult{}, apperr.Wrap("fs.token_validation", "runtime", 1, "prepare temporary FS token validation state", err) + } + defer os.RemoveAll(validationHome) + runner := s.drive9Runner() + runner.HomeDir = validationHome + if _, err := runner.Run(ctx, fswrap.RunOptions{ + Profile: &selected, + ResourceName: tokenID, + Args: []string{"fs", "stat", "--output", "json", ":/"}, + CaptureStdout: true, + IncludeFSAPIKey: true, + }); err != nil { + return ImportFileSystemTokenResult{}, err + } + status := "validated" + stored := false + if persist { + homeDir, err := s.homeDir() + if err != nil { + return ImportFileSystemTokenResult{}, err + } + if _, err := fscred.StoreCredential(homeDir, opts.Profile, tokenID, placement.Code, opts.Token, opts.Replace); err != nil { + return ImportFileSystemTokenResult{}, err + } + status = "imported" + stored = true + } + return ImportFileSystemTokenResult{FileSystemID: tokenID, RegionCode: placement.Code, CredentialsStored: stored, Status: status}, nil +} + +func (s Service) waitUntilFileSystemReady(ctx context.Context, homeDir string, profile *config.Profile, fileSystemID string) error { + selected, _, err := fscred.ResolveCredential(homeDir, profile, fscred.ResolveCredentialOptions{FileSystemID: fileSystemID, FileSystemIDExplicit: true, TokenRequired: true}) if err != nil { return err } @@ -292,7 +474,7 @@ func (s Service) waitUntilFileSystemReady(ctx context.Context, homeDir string, p if err == nil { return nil } - if waitErr := fsReadyWaitContextError(ctx, waitCtx, name, timeout); waitErr != nil { + if waitErr := fsReadyWaitContextError(ctx, waitCtx, fileSystemID, timeout); waitErr != nil { return waitErr } if !isDrive9ReadinessError(err) { @@ -300,14 +482,14 @@ func (s Service) waitUntilFileSystemReady(ctx context.Context, homeDir string, p "fs.ready_wait_failed", "runtime", 1, - fmt.Sprintf("ti fs resource %q was provisioned and its credentials were stored, but its Drive9 data plane readiness check failed", name), + fmt.Sprintf("ti fs resource %q was provisioned and its credentials were stored, but its Drive9 data plane readiness check failed", fileSystemID), err, ) } select { case <-waitCtx.Done(): - return fsReadyWaitContextError(ctx, waitCtx, name, timeout) + return fsReadyWaitContextError(ctx, waitCtx, fileSystemID, timeout) case <-ticker.C: } } @@ -374,11 +556,11 @@ func (s Service) drive9CheckFileSystem(ctx context.Context, opts CheckFileSystem checks := []Check{ {Name: "config_and_credentials", Status: "passed", Message: fmt.Sprintf("ti fs credentials for profile namespace %q loaded", profileName(opts.Profile))}, } - resource := fscred.FromProfile(opts.Profile) - if resource.Name == "" || !resource.HasAPIKey { - checks = append(checks, Check{Name: "fs_resource_credentials", Status: "warning", Message: "ti fs resource name or FS token is missing"}) + resource := fscred.Credential{FileSystemID: opts.Profile.FSTenantID, RegionCode: opts.Profile.FSPlacementRegionCode, HasLocalToken: strings.TrimSpace(opts.Profile.FSAPIKey) != "", APIKey: opts.Profile.FSAPIKey} + if resource.FileSystemID == "" || !resource.HasLocalToken { + checks = append(checks, Check{Name: "fs_resource_credentials", Status: "warning", Message: "ti fs file system ID or token is missing"}) } else { - checks = append(checks, Check{Name: "fs_resource_credentials", Status: "passed", Message: resource.Name}) + checks = append(checks, Check{Name: "fs_resource_credentials", Status: "passed", Message: resource.FileSystemID}) } endpoint, err := s.resolveFS(opts.Profile) if err != nil { @@ -391,15 +573,15 @@ func (s Service) drive9CheckFileSystem(ctx context.Context, opts CheckFileSystem return checkResult(opts.Profile, resource, &endpoint, nil, checks), nil } checks = append(checks, Check{Name: "companion_binary", Status: "passed", Message: "ti-drive9"}) - if !resource.HasAPIKey { - checks = append(checks, Check{Name: "remote_status", Status: "warning", Message: "remote status requires fs_api_key; run ti fs create-file-system first"}) + if !resource.HasLocalToken { + checks = append(checks, Check{Name: "remote_status", Status: "warning", Message: "remote status requires an FS token; create or import local credentials first"}) return checkResult(opts.Profile, resource, &endpoint, nil, checks), nil } if _, err := s.drive9Run(ctx, opts.Profile, []string{"fs", "stat", "--output", "json", ":/"}, true); err != nil { checks = append(checks, Check{Name: "remote_status", Status: "failed", Message: apperr.MessageFor(err)}) return checkResult(opts.Profile, resource, &endpoint, nil, checks), nil } - remote := apifs.StatusResponse{Status: "reachable", TenantID: resource.TenantID, Kind: "ti fs"} + remote := apifs.StatusResponse{Status: "reachable", TenantID: resource.FileSystemID, Kind: "ti fs"} checks = append(checks, Check{Name: "remote_status", Status: "passed", Message: "reachable"}) return checkResult(opts.Profile, resource, &endpoint, &remote, checks), nil } @@ -1366,6 +1548,7 @@ func (s Service) drive9MountLocatorProfile(base *config.Profile, mountPath strin profile.Name = locator.Profile profile.HomeDir = homeDir profile.FSResourceName = locator.FileSystemName + profile.FSTenantID = locator.FileSystemName profile.FSPlacementRegionCode = placement.Code profile.FSCloudProvider = placement.Provider profile.FSRegionCode = placement.NativeCode @@ -1516,6 +1699,10 @@ func isDrive9NotFound(err error) bool { return strings.Contains(strings.ToLower(err.Error()), "not found") } +func remoteFileSystemNotFound(fileSystemID string, cause error) error { + return apperr.Wrap("fs.resource_not_found", "runtime", 1, fmt.Sprintf("file system %q was not found in the selected region", fileSystemID), cause) +} + func isTransientDrive9Error(err error) bool { if err == nil { return false diff --git a/internal/fs/drive9_companion_test.go b/internal/fs/drive9_companion_test.go index 340155a..8533e50 100644 --- a/internal/fs/drive9_companion_test.go +++ b/internal/fs/drive9_companion_test.go @@ -3,6 +3,7 @@ package fs import ( "bufio" "context" + "encoding/base64" "encoding/json" "fmt" "os" @@ -35,13 +36,12 @@ func TestDrive9CreateFileSystemStoresRegistryCredentialsAndUsesCanonicalRegion(t profile := testProfile() result, err := testCompanionService(home, companion).CreateFileSystem(context.Background(), CreateFileSystemOptions{ - Profile: profile, - FileSystemName: "workspace", + Profile: profile, }) if err != nil { t.Fatalf("CreateFileSystem failed: %v", err) } - if result.FileSystemName != "workspace" || result.TenantID != "tenant-1" || result.RegionCode != "aws-us-east-1" || result.FSToken != "fs-secret" || !result.CredentialsStored { + if result.FileSystemID != "tenant-1" || result.RegionCode != "aws-us-east-1" || result.FSToken != "fs-secret" || !result.CredentialsStored { t.Fatalf("unexpected result: %#v", result) } @@ -59,16 +59,16 @@ func TestDrive9CreateFileSystemStoresRegistryCredentialsAndUsesCanonicalRegion(t if got := credentialsDoc["stage"]; got.FSAPIKey != "" { t.Fatalf("fs api key must not be stored flat under profile: %#v", got) } - resource, err := fscred.Get(home, "stage", "workspace") + resource, err := fscred.GetCredential(home, "stage", "tenant-1") if err != nil { - t.Fatalf("Get registry resource failed: %v", err) + t.Fatalf("Get ID-keyed credential failed: %v", err) } - if resource.TenantID != "tenant-1" || resource.RegionCode != "aws-us-east-1" || resource.APIKey != "fs-secret" { - t.Fatalf("unexpected registry resource: %#v", resource) + if resource.FileSystemID != "tenant-1" || resource.RegionCode != "aws-us-east-1" || resource.APIKey != "fs-secret" { + t.Fatalf("unexpected ID-keyed credential: %#v", resource) } createCall := requireFakeDrive9Call(t, recordPath, "create") - wantArgs := []string{"create", "--json", "--name", "workspace", "--region-code", "aws-us-east-1"} + wantArgs := []string{"create", "--json", "--region-code", "aws-us-east-1"} if fmt.Sprint(createCall.Args) != fmt.Sprint(wantArgs) { t.Fatalf("create args = %#v, want %#v", createCall.Args, wantArgs) } @@ -81,12 +81,12 @@ func TestDrive9CreateFileSystemStoresRegistryCredentialsAndUsesCanonicalRegion(t if _, ok := createCall.Env["DRIVE9_API_KEY"]; ok { t.Fatalf("create should not pass an fs api key, env=%#v", createCall.Env) } - wantHome, err := fscred.CompanionHome(home, "stage", "workspace") - if err != nil { - t.Fatal(err) + createHome := createCall.Env["HOME"] + if createHome == "" || strings.HasPrefix(createHome, filepath.Join(home, ".ti")) { + t.Fatalf("create HOME = %q, want isolated temporary state", createHome) } - if createCall.Env["HOME"] != wantHome { - t.Fatalf("create HOME = %q, want %q", createCall.Env["HOME"], wantHome) + if _, err := os.Stat(createHome); !os.IsNotExist(err) { + t.Fatalf("temporary create HOME was not removed: %q, err=%v", createHome, err) } } @@ -101,7 +101,6 @@ func TestDrive9CreateFileSystemWaitsUntilReady(t *testing.T) { service.FSReadyWaitPollInterval = time.Millisecond result, err := service.CreateFileSystem(context.Background(), CreateFileSystemOptions{ Profile: testProfile(), - FileSystemName: "workspace", WaitUntilReady: true, }) if err != nil { @@ -134,13 +133,12 @@ func TestDrive9CreateFileSystemReadyTimeoutPreservesCredentials(t *testing.T) { service.FSReadyWaitPollInterval = time.Millisecond _, err := service.CreateFileSystem(context.Background(), CreateFileSystemOptions{ Profile: testProfile(), - FileSystemName: "workspace", WaitUntilReady: true, }) if apperr.CodeFor(err) != "fs.ready_wait_timeout" { t.Fatalf("unexpected error: %v", err) } - resource, getErr := fscred.Get(home, "stage", "workspace") + resource, getErr := fscred.GetCredential(home, "stage", "tenant-1") if getErr != nil || resource.APIKey != "fs-secret" { t.Fatalf("readiness timeout removed stored credentials: resource=%#v err=%v", resource, getErr) } @@ -161,8 +159,7 @@ func TestDrive9CreateFileSystemFromEnvironmentProfileStoresDefaultProfile(t *tes } if _, err := testCompanionService(home, companion).CreateFileSystem(context.Background(), CreateFileSystemOptions{ - Profile: profile, - FileSystemName: "workspace", + Profile: profile, }); err != nil { t.Fatalf("CreateFileSystem failed: %v", err) } @@ -189,47 +186,53 @@ func TestDrive9CreateFileSystemFromEnvironmentProfileStoresDefaultProfile(t *tes } } -func TestDrive9CreateSecondFileSystemUsesIndependentCompanionHome(t *testing.T) { +func TestDrive9CreateAlwaysInvokesRemoteAndStoresByReturnedID(t *testing.T) { home := t.TempDir() companion, recordPath := buildFakeDrive9(t) t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) profile := testProfile() service := testCompanionService(home, companion) - if _, err := service.CreateFileSystem(context.Background(), CreateFileSystemOptions{Profile: profile, FileSystemName: "workspace"}); err != nil { - t.Fatalf("create workspace: %v", err) + if _, err := service.CreateFileSystem(context.Background(), CreateFileSystemOptions{Profile: profile}); err != nil { + t.Fatalf("first create: %v", err) } - if _, err := service.CreateFileSystem(context.Background(), CreateFileSystemOptions{Profile: profile, FileSystemName: "scratch"}); err != nil { - t.Fatalf("create scratch: %v", err) + if _, err := service.CreateFileSystem(context.Background(), CreateFileSystemOptions{Profile: profile}); err != nil { + t.Fatalf("second create: %v", err) } - repeated, err := service.CreateFileSystem(context.Background(), CreateFileSystemOptions{Profile: profile, FileSystemName: "scratch"}) + repeated, err := service.CreateFileSystem(context.Background(), CreateFileSystemOptions{Profile: profile}) if err != nil { t.Fatalf("repeat create scratch: %v", err) } - if repeated.Status != "exists" || repeated.FSToken != "fs-secret" || !repeated.CredentialsStored { + if repeated.Status != "active" || repeated.FSToken != "fs-secret" || !repeated.CredentialsStored { t.Fatalf("unexpected repeated create result: %#v", repeated) } - resources, err := fscred.List(home, profile.Name) - if err != nil || len(resources) != 2 { - t.Fatalf("resources=%#v err=%v", resources, err) + credentials, err := fscred.ListCredentials(home, profile.Name) + if err != nil || len(credentials) != 1 { + t.Fatalf("credentials=%#v err=%v", credentials, err) } calls := readFakeDrive9Calls(t, recordPath) homes := map[string]bool{} for _, call := range calls { - if len(call.Args) > 0 && call.Args[0] == "create" { + if hasArgPrefix(call.Args, []string{"create"}) { homes[call.Env["HOME"]] = true + if strings.HasPrefix(call.Env["HOME"], filepath.Join(home, ".ti")) { + t.Fatalf("create used persistent companion HOME: %q", call.Env["HOME"]) + } + if _, err := os.Stat(call.Env["HOME"]); !os.IsNotExist(err) { + t.Fatalf("temporary create HOME was not removed: %q, err=%v", call.Env["HOME"], err) + } } } - if len(homes) != 2 { - t.Fatalf("expected two resource-scoped companion homes, got %#v", homes) + if len(homes) != 3 { + t.Fatalf("expected an isolated companion home for each create, got %#v", homes) } createCalls := 0 for _, call := range calls { - if len(call.Args) > 0 && call.Args[0] == "create" { + if hasArgPrefix(call.Args, []string{"create"}) { createCalls++ } } - if createCalls != 2 { - t.Fatalf("idempotent create invoked Drive9 %d times, want 2 total calls", createCalls) + if createCalls != 3 { + t.Fatalf("create invoked Drive9 %d times, want 3 total calls", createCalls) } } @@ -241,15 +244,15 @@ func TestDrive9DeleteFileSystemDeletesOnlySelectedRegistryResource(t *testing.T) if err := store.WriteProfile(home, profile.Name, store.ConfigProfile{RegionCode: profile.PlacementRegionCode}, store.CredentialsProfile{TiDBCloudPublicKey: profile.TiDBCloudPublicKey, TiDBCloudPrivateKey: profile.TiDBCloudPrivateKey}); err != nil { t.Fatal(err) } - if err := fscred.Store(home, profile, "workspace", "tenant-1", "aws", "aws-us-east-1", "fs-secret"); err != nil { + if _, err := fscred.StoreCredential(home, profile, "tenant-1", "aws-us-east-1", "fs-secret", false); err != nil { t.Fatal(err) } - if err := fscred.Store(home, profile, "scratch", "tenant-2", "aws", "aws-us-east-1", "fs-secret-2"); err != nil { + if _, err := fscred.StoreCredential(home, profile, "tenant-2", "aws-us-east-1", "fs-secret-2", false); err != nil { t.Fatal(err) } result, err := testCompanionService(home, companion).DeleteFileSystem(context.Background(), DeleteFileSystemOptions{ - Profile: profile, - FileSystemName: "workspace", + Profile: profile, + FileSystemID: "tenant-1", }) if err != nil { t.Fatalf("DeleteFileSystem failed: %v", err) @@ -257,11 +260,11 @@ func TestDrive9DeleteFileSystemDeletesOnlySelectedRegistryResource(t *testing.T) if !result.CredentialsRemoved || result.Status != "deleting" || result.RemoteDeletionState != "deleting" { t.Fatalf("unexpected delete result: %#v", result) } - deleteCall := requireFakeDrive9Call(t, recordPath, "delete") - if fmt.Sprint(deleteCall.Args) != fmt.Sprint([]string{"delete", "--json", "--yes"}) { + deleteCall := requireFakeDrive9Call(t, recordPath, "admin", "tenant", "delete") + if fmt.Sprint(deleteCall.Args) != fmt.Sprint([]string{"admin", "tenant", "delete", "--json", "--region-code", "aws-us-east-1", "--tenant-id", "tenant-1"}) { t.Fatalf("delete args = %#v", deleteCall.Args) } - if deleteCall.Env["DRIVE9_API_KEY"] != "fs-secret" || deleteCall.Env["DRIVE9_PUBLIC_KEY"] != "public" || deleteCall.Env["DRIVE9_PRIVATE_KEY"] != "private" { + if deleteCall.Env["DRIVE9_API_KEY"] != "" || deleteCall.Env["DRIVE9_PUBLIC_KEY"] != "public" || deleteCall.Env["DRIVE9_PRIVATE_KEY"] != "private" { t.Fatalf("missing delete env: %#v", deleteCall.Env) } @@ -279,10 +282,10 @@ func TestDrive9DeleteFileSystemDeletesOnlySelectedRegistryResource(t *testing.T) if got := credentialsDoc["stage"]; got.FSAPIKey != "" || got.TiDBCloudPublicKey != "public" { t.Fatalf("unexpected credentials after delete: %#v", got) } - if _, err := fscred.Get(home, "stage", "workspace"); apperr.CodeFor(err) != "fs.resource_not_found" { + if _, err := fscred.GetCredential(home, "stage", "tenant-1"); apperr.CodeFor(err) != "fs.credential_not_found" { t.Fatalf("deleted resource still exists: %v", err) } - if resource, err := fscred.Get(home, "stage", "scratch"); err != nil || resource.APIKey != "fs-secret-2" { + if resource, err := fscred.GetCredential(home, "stage", "tenant-2"); err != nil || resource.APIKey != "fs-secret-2" { t.Fatalf("unrelated resource was changed: resource=%#v err=%v", resource, err) } } @@ -318,6 +321,407 @@ func TestDrive9CheckFileSystemUsesSelectedResource(t *testing.T) { } } +func TestDrive9RemoteInventoryAndDescribeJoinLocalToken(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + t.Setenv("TI_FAKE_DRIVE9_LIST_MODE", "mixed-local-token") + profile := testProfile() + if _, err := fscred.StoreCredential(home, profile, "tenant-1", "aws-us-east-1", fsTestToken(t, "tenant-1"), false); err != nil { + t.Fatal(err) + } + service := testCompanionService(home, companion) + list, err := service.ListFileSystems(context.Background(), profile) + if err != nil { + t.Fatal(err) + } + if len(list.FileSystems) != 2 || list.FileSystems[0].FileSystemID != "tenant-1" || !list.FileSystems[0].HasLocalToken { + t.Fatalf("list = %#v", list) + } + if list.FileSystems[1].FileSystemID != "tenant-2" || list.FileSystems[1].HasLocalToken { + t.Fatalf("remote resource without a local token was not preserved: %#v", list) + } + described, err := service.DescribeFileSystem(context.Background(), profile, "tenant-1") + if err != nil { + t.Fatal(err) + } + if described.FileSystemID != "tenant-1" || described.Status != "active" || !described.HasLocalToken { + t.Fatalf("described = %#v", described) + } + listCall := requireFakeDrive9Call(t, recordPath, "admin", "tenant", "list") + if listCall.Env["DRIVE9_API_KEY"] != "" || listCall.Env["DRIVE9_PUBLIC_KEY"] != "public" { + t.Fatalf("inventory used wrong credentials: %#v", listCall.Env) + } +} + +func TestDrive9RemoteInventoryCommandsRejectMissingTiDBCloudCredentials(t *testing.T) { + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + profile := testProfile() + profile.TiDBCloudPublicKey = "" + profile.TiDBCloudPrivateKey = "" + service := testCompanionService(t.TempDir(), companion) + + tests := []struct { + name string + run func() error + }{ + {name: "list", run: func() error { + _, err := service.ListFileSystems(context.Background(), profile) + return err + }}, + {name: "describe", run: func() error { + _, err := service.DescribeFileSystem(context.Background(), profile, "tenant-1") + return err + }}, + {name: "delete", run: func() error { + _, err := service.DeleteFileSystem(context.Background(), DeleteFileSystemOptions{Profile: profile, FileSystemID: "tenant-1"}) + return err + }}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if err := tc.run(); apperr.CodeFor(err) != "auth.missing_credentials" { + t.Fatalf("error = %v, want auth.missing_credentials", err) + } + }) + } + if _, err := os.Stat(recordPath); !os.IsNotExist(err) { + t.Fatalf("missing credentials invoked Drive9: %v", err) + } +} + +func TestDrive9DescribeMigratesLegacyCredentialBeforeJoiningLocalToken(t *testing.T) { + home := t.TempDir() + companion, _ := buildFakeDrive9(t) + profile := testProfile() + if err := fscred.Store(home, profile, "workspace", "tenant-1", "aws", "aws-us-east-1", fsTestToken(t, "tenant-1")); err != nil { + t.Fatal(err) + } + + described, err := testCompanionService(home, companion).DescribeFileSystem(context.Background(), profile, "tenant-1") + if err != nil { + t.Fatal(err) + } + if !described.HasLocalToken { + t.Fatalf("describe did not join the migrated legacy credential: %#v", described) + } + if _, err := fscred.GetCredential(home, profile.Name, "tenant-1"); err != nil { + t.Fatalf("describe did not migrate the legacy credential: %v", err) + } +} + +func TestDrive9DeleteMigratesLegacyCredentialAndDoesNotRestoreIt(t *testing.T) { + home := t.TempDir() + companion, _ := buildFakeDrive9(t) + profile := testProfile() + if err := fscred.Store(home, profile, "workspace", "tenant-1", "aws", "aws-us-east-1", fsTestToken(t, "tenant-1")); err != nil { + t.Fatal(err) + } + + result, err := testCompanionService(home, companion).DeleteFileSystem(context.Background(), DeleteFileSystemOptions{Profile: profile, FileSystemID: "tenant-1"}) + if err != nil { + t.Fatal(err) + } + if !result.CredentialsRemoved { + t.Fatalf("delete did not remove the migrated credential: %#v", result) + } + if err := fscred.MigrateNameRegistry(home, profile); err != nil { + t.Fatal(err) + } + if _, err := fscred.GetCredential(home, profile.Name, "tenant-1"); apperr.CodeFor(err) != "fs.credential_not_found" { + t.Fatalf("legacy rollback source restored the deleted credential: %v", err) + } + if _, err := fscred.Get(home, profile.Name, "workspace"); err != nil { + t.Fatalf("delete removed the legacy rollback source: %v", err) + } +} + +func TestDrive9RemoteInventoryPaginationSortingAndEmptyResults(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + t.Setenv("TI_FAKE_DRIVE9_LIST_MODE", "paginate") + profile := testProfile() + service := testCompanionService(home, companion) + + result, err := service.ListFileSystems(context.Background(), profile) + if err != nil { + t.Fatal(err) + } + if len(result.FileSystems) != 2 || result.FileSystems[0].FileSystemID != "tenant-1" || result.FileSystems[1].FileSystemID != "tenant-2" { + t.Fatalf("paginated inventory was not sorted: %#v", result.FileSystems) + } + listCalls := 0 + for _, call := range readFakeDrive9Calls(t, recordPath) { + if hasArgPrefix(call.Args, []string{"admin", "tenant", "list"}) { + listCalls++ + if !containsArg(call.Args, "--page-size") || call.Env["DRIVE9_REGION_CODE"] != "aws-us-east-1" { + t.Fatalf("inventory call did not preserve pagination or region routing: %#v", call) + } + } + } + if listCalls != 2 { + t.Fatalf("inventory list calls = %d, want 2", listCalls) + } + + t.Setenv("TI_FAKE_DRIVE9_LIST_MODE", "empty") + empty, err := service.ListFileSystems(context.Background(), profile) + if err != nil { + t.Fatal(err) + } + if empty.FileSystems == nil || len(empty.FileSystems) != 0 { + t.Fatalf("empty inventory = %#v, want an empty JSON array", empty.FileSystems) + } +} + +func TestDrive9RemoteInventoryRejectsInvalidResponses(t *testing.T) { + for _, tc := range []struct { + name string + mode string + }{ + {name: "malformed JSON", mode: "malformed"}, + {name: "regressing next page", mode: "regress"}, + {name: "mismatched response page", mode: "page-mismatch"}, + {name: "duplicate file system ID", mode: "duplicate"}, + } { + t.Run(tc.name, func(t *testing.T) { + companion, _ := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_LIST_MODE", tc.mode) + _, err := testCompanionService(t.TempDir(), companion).ListFileSystems(context.Background(), testProfile()) + if apperr.CodeFor(err) != "fs.companion_decode" { + t.Fatalf("inventory error = %v, want fs.companion_decode", err) + } + }) + } +} + +func TestDrive9CreateReturnsOneTimeTokenWhenLocalPersistenceFails(t *testing.T) { + home := t.TempDir() + companion, _ := buildFakeDrive9(t) + paths, err := fscred.CredentialPath(home, "stage", "tenant-1") + if err != nil { + t.Fatal(err) + } + profileCredentialDir := filepath.Dir(filepath.Dir(paths.Credentials)) + t.Setenv("TI_FAKE_DRIVE9_BREAK_CREDENTIAL_ROOT", profileCredentialDir) + var stderr strings.Builder + service := testCompanionService(home, companion) + service.Stderr = &stderr + result, err := service.CreateFileSystem(context.Background(), CreateFileSystemOptions{Profile: testProfile()}) + if err != nil { + t.Fatalf("remote create should remain successful after local persistence failure: %v", err) + } + if result.FileSystemID != "tenant-1" || result.FSToken != "fs-secret" || result.CredentialsStored { + t.Fatalf("create result lost one-time recovery data: %#v", result) + } + if !strings.Contains(stderr.String(), "was created") || strings.Contains(stderr.String(), result.FSToken) { + t.Fatalf("create warning is missing or leaked the token: %q", stderr.String()) + } +} + +func TestDrive9CreatePreflightRejectsUnwritableCredentialStoreBeforeRemoteCall(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + credentialRoot := filepath.Join(home, ".ti", "fs_credentials") + if err := os.MkdirAll(filepath.Dir(credentialRoot), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(credentialRoot, []byte("not a directory"), 0o600); err != nil { + t.Fatal(err) + } + _, err := testCompanionService(home, companion).CreateFileSystem(context.Background(), CreateFileSystemOptions{Profile: testProfile()}) + if err == nil { + t.Fatal("create should reject an unusable local credential store") + } + if _, statErr := os.Stat(recordPath); !os.IsNotExist(statErr) { + t.Fatalf("credential preflight failure invoked Drive9 or created its record: %v", statErr) + } +} + +func TestDrive9DeleteFailurePreservesLocalCredential(t *testing.T) { + home := t.TempDir() + companion, _ := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_DELETE_FAIL", "1") + profile := testProfile() + if _, err := fscred.StoreCredential(home, profile, "tenant-1", "aws-us-east-1", "fs-secret", false); err != nil { + t.Fatal(err) + } + _, err := testCompanionService(home, companion).DeleteFileSystem(context.Background(), DeleteFileSystemOptions{Profile: profile, FileSystemID: "tenant-1"}) + if err == nil { + t.Fatal("delete should return the remote failure") + } + credential, getErr := fscred.GetCredential(home, profile.Name, "tenant-1") + if getErr != nil || credential.APIKey != "fs-secret" { + t.Fatalf("remote delete failure removed local credentials: credential=%#v err=%v", credential, getErr) + } +} + +func TestDrive9DescribeAndDeleteMapRemoteNotFound(t *testing.T) { + companion, _ := buildFakeDrive9(t) + service := testCompanionService(t.TempDir(), companion) + t.Setenv("TI_FAKE_DRIVE9_NOT_FOUND", "1") + if _, err := service.DescribeFileSystem(context.Background(), testProfile(), "tenant-missing"); apperr.CodeFor(err) != "fs.resource_not_found" { + t.Fatalf("describe error = %v, want fs.resource_not_found", err) + } + if _, err := service.DeleteFileSystem(context.Background(), DeleteFileSystemOptions{Profile: testProfile(), FileSystemID: "tenant-missing"}); apperr.CodeFor(err) != "fs.resource_not_found" { + t.Fatalf("delete error = %v, want fs.resource_not_found", err) + } +} + +func TestImportFileSystemTokenValidatesStatusAndStoresCredential(t *testing.T) { + token := fsTestToken(t, "tenant-import") + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + t.Setenv("TI_FAKE_DRIVE9_EXPECT_API_KEY", token) + profile := testProfile() + profile.HomeDir = home + service := testCompanionService(home, companion) + result, err := service.ImportFileSystemToken(context.Background(), ImportFileSystemTokenOptions{Profile: profile, Token: token}) + if err != nil { + t.Fatal(err) + } + if result.FileSystemID != "tenant-import" || !result.CredentialsStored || result.Status != "imported" { + t.Fatalf("result = %#v", result) + } + credential, err := fscred.GetCredential(home, profile.Name, "tenant-import") + if err != nil || credential.APIKey != token { + t.Fatalf("credential=%#v err=%v", credential, err) + } + validationCall := requireFakeDrive9Call(t, recordPath, "fs", "stat") + if validationCall.Env["DRIVE9_API_KEY"] != token { + t.Fatalf("token validation used wrong credential: %#v", validationCall.Env) + } + if strings.HasPrefix(validationCall.Env["HOME"], filepath.Join(home, store.TIDirName)) { + t.Fatalf("token validation persisted companion state under the ti home: %#v", validationCall.Env) + } + if _, err := os.Stat(validationCall.Env["HOME"]); !os.IsNotExist(err) { + t.Fatalf("temporary token validation HOME was not removed: %q, err=%v", validationCall.Env["HOME"], err) + } +} + +func TestImportFileSystemTokenRejectsRemoteValidationFailureWithoutWriting(t *testing.T) { + home := t.TempDir() + companion, _ := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_EXPECT_API_KEY", fsTestToken(t, "tenant-other")) + profile := testProfile() + profile.HomeDir = home + _, err := testCompanionService(home, companion).ImportFileSystemToken(context.Background(), ImportFileSystemTokenOptions{ + Profile: profile, + Token: fsTestToken(t, "tenant-rejected"), + }) + if err == nil { + t.Fatal("import should reject a token refused by Drive9") + } + if _, getErr := fscred.GetCredential(home, profile.Name, "tenant-rejected"); apperr.CodeFor(getErr) != "fs.credential_not_found" { + t.Fatalf("rejected import wrote credentials: %v", getErr) + } +} + +func TestImportFileSystemTokenRejectsAssertedIDMismatchBeforeRemoteValidation(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + profile := testProfile() + profile.HomeDir = home + _, err := testCompanionService(home, companion).ImportFileSystemToken(context.Background(), ImportFileSystemTokenOptions{ + Profile: profile, + FileSystemID: "tenant-other", + Token: fsTestToken(t, "tenant-import"), + }) + if apperr.CodeFor(err) != "fs.token_file_system_mismatch" { + t.Fatalf("error = %v, want fs.token_file_system_mismatch", err) + } + if _, statErr := os.Stat(recordPath); !os.IsNotExist(statErr) { + t.Fatalf("ID mismatch invoked Drive9: %v", statErr) + } + if _, getErr := fscred.GetCredential(home, profile.Name, "tenant-import"); apperr.CodeFor(getErr) != "fs.credential_not_found" { + t.Fatalf("ID mismatch wrote credentials: %v", getErr) + } +} + +func TestImportFileSystemTokenIsIdempotentWithoutRewritingCredential(t *testing.T) { + home := t.TempDir() + companion, _ := buildFakeDrive9(t) + token := fsTestToken(t, "tenant-import") + t.Setenv("TI_FAKE_DRIVE9_EXPECT_API_KEY", token) + profile := testProfile() + profile.HomeDir = home + service := testCompanionService(home, companion) + opts := ImportFileSystemTokenOptions{Profile: profile, Token: token} + if _, err := service.ImportFileSystemToken(context.Background(), opts); err != nil { + t.Fatal(err) + } + paths, err := fscred.CredentialPath(home, profile.Name, "tenant-import") + if err != nil { + t.Fatal(err) + } + wantModTime := time.Unix(1, 0) + if err := os.Chtimes(paths.Credentials, wantModTime, wantModTime); err != nil { + t.Fatal(err) + } + if _, err := service.ImportFileSystemToken(context.Background(), opts); err != nil { + t.Fatal(err) + } + info, err := os.Stat(paths.Credentials) + if err != nil { + t.Fatal(err) + } + if !info.ModTime().Equal(wantModTime) { + t.Fatalf("idempotent import rewrote credentials: modtime=%s", info.ModTime()) + } +} + +func TestImportFileSystemTokenReplaceCanUpdateStoredRegionAfterValidation(t *testing.T) { + home := t.TempDir() + companion, _ := buildFakeDrive9(t) + oldToken := fsTestTokenVariant(t, "tenant-import", "old") + newToken := fsTestTokenVariant(t, "tenant-import", "new") + profile := testProfile() + profile.HomeDir = home + if _, err := fscred.StoreCredential(home, profile, "tenant-import", "aws-us-east-1", oldToken, false); err != nil { + t.Fatal(err) + } + profile.PlacementRegionCode = "aws-us-west-2" + profile.RegionCode = "us-west-2" + t.Setenv("TI_FAKE_DRIVE9_EXPECT_API_KEY", newToken) + resolver := endpoints.Resolver{FSManifest: &endpoints.FSRegionManifest{Regions: []endpoints.FSRegionManifestEntry{ + {RegionCode: "aws-us-west-2", Mode: endpoints.DefaultFSMode, ServerURL: "https://fs-west.test", CloudProvider: "aws", TiDBRegion: "us-west-2"}, + }}} + service := Service{HomeDir: home, CompanionPath: companion, Resolver: resolver} + if _, err := service.ImportFileSystemToken(context.Background(), ImportFileSystemTokenOptions{Profile: profile, Token: newToken}); apperr.CodeFor(err) != "fs.credential_import_conflict" { + t.Fatalf("import conflict error = %v", err) + } + result, err := service.ImportFileSystemToken(context.Background(), ImportFileSystemTokenOptions{Profile: profile, Token: newToken, Replace: true}) + if err != nil { + t.Fatal(err) + } + if result.RegionCode != "aws-us-west-2" { + t.Fatalf("result = %#v", result) + } + credential, err := fscred.GetCredential(home, profile.Name, "tenant-import") + if err != nil || credential.APIKey != newToken || credential.RegionCode != "aws-us-west-2" { + t.Fatalf("credential=%#v err=%v", credential, err) + } +} + +func fsTestToken(t *testing.T, tenantID string) string { + return fsTestTokenVariant(t, tenantID, "signature") +} + +func fsTestTokenVariant(t *testing.T, tenantID, signature string) string { + t.Helper() + header := base64.RawURLEncoding.EncodeToString([]byte(`{"alg":"HS256"}`)) + payloadBytes, err := json.Marshal(map[string]any{"tenant_id": tenantID, "token_version": 1, "iat": 1}) + if err != nil { + t.Fatal(err) + } + jwt := header + "." + base64.RawURLEncoding.EncodeToString(payloadBytes) + "." + signature + return "drive9_" + base64.RawURLEncoding.EncodeToString([]byte(jwt)) +} + func TestDrive9DataPlaneCommandsTranslateToCompanion(t *testing.T) { home := t.TempDir() companion, recordPath := buildFakeDrive9(t) @@ -546,7 +950,6 @@ func TestDryRunCreateFileSystemUsesRedactedProvisionShape(t *testing.T) { profile := testProfile() result, err := Service{Resolver: supportedFSManifestResolver("https://fs.test")}.DryRunCreateFileSystem(context.Background(), "ti fs create-file-system", CreateFileSystemOptions{ Profile: profile, - FileSystemName: "workspace", WaitUntilReady: true, }) if err != nil { @@ -577,34 +980,34 @@ func TestDryRunCreateFileSystemUsesRedactedProvisionShape(t *testing.T) { } } -func TestDryRunDeleteFileSystemReportsRegistryFiles(t *testing.T) { +func TestDryRunDeleteFileSystemReportsCredentialFile(t *testing.T) { home := t.TempDir() profile := dataProfile() - if err := fscred.Store(home, profile, "workspace", "tenant-1", "aws", "aws-us-east-1", "fs-secret"); err != nil { + if _, err := fscred.StoreCredential(home, profile, "tenant-1", "aws-us-east-1", "fs-secret", false); err != nil { t.Fatal(err) } result, err := (Service{HomeDir: home, Resolver: supportedFSManifestResolver("https://fs.test")}).DryRunDeleteFileSystem(context.Background(), "ti fs delete-file-system", DeleteFileSystemOptions{ - Profile: profile, - FileSystemName: "workspace", + Profile: profile, + FileSystemID: "tenant-1", }) if err != nil { t.Fatalf("DryRunDeleteFileSystem failed: %v", err) } - paths, err := fscred.Paths(home, profile.Name, "workspace") + paths, err := fscred.CredentialPath(home, profile.Name, "tenant-1") if err != nil { t.Fatal(err) } found := false for _, check := range result.Checks { - if check.Name == "local_resource_registry" { - found = strings.Contains(check.Message, paths.Config) && strings.Contains(check.Message, paths.Credentials) + if check.Name == "local_credentials" { + found = strings.Contains(check.Message, paths.Credentials) } } if !found { t.Fatalf("dry-run did not report registry files: %#v", result.Checks) } - if _, err := fscred.Get(home, profile.Name, "workspace"); err != nil { - t.Fatalf("dry-run removed registry resource: %v", err) + if _, err := fscred.GetCredential(home, profile.Name, "tenant-1"); err != nil { + t.Fatalf("dry-run removed credential: %v", err) } } @@ -655,6 +1058,10 @@ func main() { } switch { case args[0] == "create": + if path := os.Getenv("TI_FAKE_DRIVE9_BREAK_CREDENTIAL_ROOT"); path != "" { + _ = os.RemoveAll(path) + _ = os.WriteFile(path, []byte("not a directory"), 0600) + } _ = json.NewEncoder(os.Stdout).Encode(map[string]string{ "tenant_id": "tenant-1", "api_key": "fs-secret", @@ -663,8 +1070,70 @@ func main() { "region_code": os.Getenv("DRIVE9_REGION_CODE"), "server": os.Getenv("DRIVE9_SERVER"), }) - case args[0] == "delete": - _ = json.NewEncoder(os.Stdout).Encode(map[string]string{"status": "deleting"}) + case len(args) >= 3 && args[0] == "admin" && args[1] == "tenant" && args[2] == "delete": + if os.Getenv("TI_FAKE_DRIVE9_NOT_FOUND") == "1" { + fmt.Fprintln(os.Stderr, "delete admin tenant: HTTP 404: tenant not found") + os.Exit(1) + } + if os.Getenv("TI_FAKE_DRIVE9_DELETE_FAIL") == "1" { + fmt.Fprintln(os.Stderr, "admin tenant delete: backend unavailable") + os.Exit(1) + } + _ = json.NewEncoder(os.Stdout).Encode(map[string]string{"tenant_id": "tenant-1", "status": "deleting"}) + case len(args) >= 3 && args[0] == "admin" && args[1] == "tenant" && args[2] == "list": + switch os.Getenv("TI_FAKE_DRIVE9_LIST_MODE") { + case "empty": + _ = json.NewEncoder(os.Stdout).Encode(map[string]any{"tenants": []any{}, "page": 1, "page_size": 100}) + case "mixed-local-token": + _ = json.NewEncoder(os.Stdout).Encode(map[string]any{ + "tenants": []map[string]any{ + {"tenant_id": "tenant-1", "status": "active", "kind": "live"}, + {"tenant_id": "tenant-2", "status": "active", "kind": "live"}, + }, + "page": 1, "page_size": 100, + }) + case "malformed": + fmt.Fprint(os.Stdout, "{") + case "paginate": + if flagValue(args, "--page") == "1" { + _ = json.NewEncoder(os.Stdout).Encode(map[string]any{ + "tenants": []map[string]any{{"tenant_id": "tenant-2", "status": "active", "kind": "live"}}, + "page": 1, "page_size": 100, "next_page": 2, + }) + } else { + _ = json.NewEncoder(os.Stdout).Encode(map[string]any{ + "tenants": []map[string]any{{"tenant_id": "tenant-1", "status": "active", "kind": "live"}}, + "page": 2, "page_size": 100, + }) + } + case "regress": + _ = json.NewEncoder(os.Stdout).Encode(map[string]any{ + "tenants": []any{}, "page": 1, "page_size": 100, "next_page": 1, + }) + case "page-mismatch": + _ = json.NewEncoder(os.Stdout).Encode(map[string]any{ + "tenants": []any{}, "page": 2, "page_size": 100, + }) + case "duplicate": + _ = json.NewEncoder(os.Stdout).Encode(map[string]any{ + "tenants": []map[string]any{ + {"tenant_id": "tenant-1", "status": "active", "kind": "live"}, + {"tenant_id": "tenant-1", "status": "active", "kind": "live"}, + }, + "page": 1, "page_size": 100, + }) + default: + _ = json.NewEncoder(os.Stdout).Encode(map[string]any{ + "tenants": []map[string]any{{"tenant_id": "tenant-1", "status": "active", "kind": "live"}}, + "page": 1, "page_size": 100, + }) + } + case len(args) >= 3 && args[0] == "admin" && args[1] == "tenant" && args[2] == "get": + if os.Getenv("TI_FAKE_DRIVE9_NOT_FOUND") == "1" { + fmt.Fprintln(os.Stderr, "get admin tenant: HTTP 404: tenant not found") + os.Exit(1) + } + _ = json.NewEncoder(os.Stdout).Encode(map[string]any{"tenant_id": "tenant-1", "status": "active", "kind": "live"}) case len(args) >= 2 && args[0] == "fs" && args[1] == "cat": fmt.Fprint(os.Stdout, "file bytes") case len(args) >= 2 && args[0] == "fs" && args[1] == "cp" && os.Getenv("TI_FAKE_DRIVE9_CP_FAILURE_SEQUENCE") != "": @@ -677,6 +1146,10 @@ func main() { fmt.Fprintln(os.Stderr, "fs cp: remote resource not found") os.Exit(1) case len(args) >= 2 && args[0] == "fs" && args[1] == "stat": + if expected := os.Getenv("TI_FAKE_DRIVE9_EXPECT_API_KEY"); expected != "" && os.Getenv("DRIVE9_API_KEY") != expected { + fmt.Fprintln(os.Stderr, "fs stat: unauthorized") + os.Exit(1) + } if os.Getenv("TI_FAKE_DRIVE9_STAT_ALWAYS_FAIL") == "1" { fmt.Fprintln(os.Stderr, "fs stat: storage backend unavailable; resource is still provisioning") os.Exit(1) @@ -712,6 +1185,15 @@ func record(args []string) { defer f.Close() _ = json.NewEncoder(f).Encode(callRecord{Args: args, Env: env}) } + +func flagValue(args []string, name string) string { + for i := 0; i+1 < len(args); i++ { + if args[i] == name { + return args[i+1] + } + } + return "" +} ` if err := os.WriteFile(sourcePath, []byte(source), 0o600); err != nil { t.Fatalf("write fake companion source: %v", err) diff --git a/internal/fs/fscred/credential.go b/internal/fs/fscred/credential.go new file mode 100644 index 0000000..1325c24 --- /dev/null +++ b/internal/fs/fscred/credential.go @@ -0,0 +1,530 @@ +package fscred + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + + "github.com/pelletier/go-toml/v2" + "github.com/tidbcloud/ti-cli/internal/apperr" + "github.com/tidbcloud/ti-cli/internal/config" + "github.com/tidbcloud/ti-cli/internal/config/region" + "github.com/tidbcloud/ti-cli/internal/config/store" +) + +const ( + credentialsDirName = "fs_credentials" + migrationStateFileName = ".legacy-name-registry-migration" + migrationStateSchema = 1 +) + +var migrationMu sync.Mutex + +type Credential struct { + FileSystemID string `json:"file_system_id" toml:"file_system_id"` + RegionCode string `json:"region_code" toml:"region_code"` + HasLocalToken bool `json:"has_local_token" toml:"-"` + APIKey string `json:"-" toml:"api_key"` +} + +type CredentialPaths struct { + Credentials string `json:"credentials"` +} + +type migrationState struct { + SchemaVersion int `toml:"schema_version"` + FileSystemIDs []string `toml:"file_system_ids"` +} + +type ResolveCredentialOptions struct { + FileSystemID string + FileSystemIDExplicit bool + Token string + TokenExplicit bool + RegionOverride string + TokenRequired bool + Env map[string]string + DryRun bool +} + +func StoreCredential(homeDir string, profile *config.Profile, fileSystemID, regionCode, apiKey string, replace bool) (Credential, error) { + if profile == nil { + return Credential{}, apperr.New("fs.missing_profile", "config", 2, "active profile is required") + } + fileSystemID, err := ValidateFileSystemID(fileSystemID) + if err != nil { + return Credential{}, err + } + apiKey = strings.TrimSpace(apiKey) + if apiKey == "" { + return Credential{}, apperr.New("fs.missing_token", "authentication", 3, "authentication required: missing FS token") + } + placementCode := strings.TrimSpace(regionCode) + if placementCode == "" { + placementCode = strings.TrimSpace(profile.PlacementRegionCode) + } + placement, err := region.ParsePlacementCode(placementCode) + if err != nil { + return Credential{}, apperr.Wrap("config.invalid_region", "config", 2, err.Error(), err) + } + credential := Credential{FileSystemID: fileSystemID, RegionCode: placement.Code, HasLocalToken: true, APIKey: apiKey} + if existing, getErr := GetCredential(homeDir, profile.Name, fileSystemID); getErr == nil { + if existing.RegionCode == credential.RegionCode && existing.APIKey == credential.APIKey { + return existing, nil + } + if !replace { + return Credential{}, credentialError("fs.credential_import_conflict", profile.Name, fileSystemID, "a different local token or region is already stored") + } + } else if apperr.CodeFor(getErr) != "fs.credential_not_found" { + return Credential{}, getErr + } + dir, err := credentialDir(homeDir, profile.Name, fileSystemID) + if err != nil { + return Credential{}, err + } + if err := ensureCredentialDirs(homeDir, profile.Name, fileSystemID); err != nil { + return Credential{}, fmt.Errorf("create ti fs credential directory: %w", err) + } + if err := writeTOML(filepath.Join(dir, credsFileName), credential, 0o600); err != nil { + return Credential{}, err + } + stored, err := GetCredential(homeDir, profile.Name, fileSystemID) + if err != nil { + return Credential{}, err + } + if stored.RegionCode != credential.RegionCode || stored.APIKey != credential.APIKey { + return Credential{}, credentialError("fs.credential_store_failed", profile.Name, fileSystemID, "stored credential verification failed") + } + return stored, nil +} + +func GetCredential(homeDir, profileName, fileSystemID string) (Credential, error) { + fileSystemID, err := ValidateFileSystemID(fileSystemID) + if err != nil { + return Credential{}, err + } + dir, err := credentialDir(homeDir, profileName, fileSystemID) + if err != nil { + return Credential{}, err + } + path := filepath.Join(dir, credsFileName) + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return Credential{}, credentialError("fs.credential_not_found", profileName, fileSystemID, "local FS credentials are not configured") + } + if err != nil { + return Credential{}, err + } + if info, statErr := os.Stat(path); statErr != nil || info.Mode().Perm()&0o077 != 0 { + if statErr == nil { + statErr = os.Chmod(path, 0o600) + } + if statErr != nil { + return Credential{}, credentialError("fs.credential_incomplete", profileName, fileSystemID, "cannot restrict credential permissions") + } + } + var credential Credential + if err := toml.Unmarshal(data, &credential); err != nil { + return Credential{}, credentialError("fs.credential_incomplete", profileName, fileSystemID, "cannot parse local credentials") + } + if credential.FileSystemID != fileSystemID || strings.TrimSpace(credential.APIKey) == "" { + return Credential{}, credentialError("fs.credential_incomplete", profileName, fileSystemID, "local credentials are incomplete") + } + placement, err := region.ParsePlacementCode(credential.RegionCode) + if err != nil { + return Credential{}, credentialError("fs.credential_incomplete", profileName, fileSystemID, "stored region_code is invalid") + } + credential.RegionCode = placement.Code + credential.APIKey = strings.TrimSpace(credential.APIKey) + credential.HasLocalToken = true + return credential, nil +} + +func ListCredentials(homeDir, profileName string) ([]Credential, error) { + dir := credentialProfileDir(homeDir, profileName) + entries, err := os.ReadDir(dir) + if errors.Is(err, os.ErrNotExist) { + return []Credential{}, nil + } + if err != nil { + return nil, err + } + credentials := make([]Credential, 0, len(entries)) + for _, entry := range entries { + if !entry.IsDir() { + continue + } + id, err := decodeKey(entry.Name()) + if err != nil { + return nil, credentialError("fs.credential_incomplete", profileName, entry.Name(), "invalid credential directory") + } + credential, err := GetCredential(homeDir, profileName, id) + if err != nil { + return nil, err + } + credentials = append(credentials, credential) + } + return credentials, nil +} + +func DeleteCredential(homeDir, profileName, fileSystemID string) (bool, error) { + if _, err := GetCredential(homeDir, profileName, fileSystemID); err != nil { + if apperr.CodeFor(err) == "fs.credential_not_found" { + return false, nil + } + return false, err + } + dir, err := credentialDir(homeDir, profileName, fileSystemID) + if err != nil { + return false, err + } + if err := os.RemoveAll(dir); err != nil { + return false, err + } + return true, nil +} + +func CredentialPath(homeDir, profileName, fileSystemID string) (CredentialPaths, error) { + dir, err := credentialDir(homeDir, profileName, fileSystemID) + if err != nil { + return CredentialPaths{}, err + } + return CredentialPaths{Credentials: filepath.Join(dir, credsFileName)}, nil +} + +func PrepareCredentialStore(homeDir, profileName string) error { + root := filepath.Join(homeDir, store.TIDirName, credentialsDirName) + profilePath := credentialProfileDir(homeDir, profileName) + for _, path := range []string{root, profilePath} { + if err := os.MkdirAll(path, 0o700); err != nil { + return fmt.Errorf("prepare ti fs credential directory: %w", err) + } + if err := os.Chmod(path, 0o700); err != nil { + return fmt.Errorf("restrict ti fs credential directory: %w", err) + } + } + probe, err := os.CreateTemp(profilePath, ".write-probe-*") + if err != nil { + return fmt.Errorf("verify ti fs credential directory is writable: %w", err) + } + probePath := probe.Name() + if closeErr := probe.Close(); closeErr != nil { + _ = os.Remove(probePath) + return fmt.Errorf("verify ti fs credential directory is writable: %w", closeErr) + } + if err := os.Remove(probePath); err != nil { + return fmt.Errorf("remove ti fs credential write probe: %w", err) + } + return nil +} + +func ResolveCredential(homeDir string, profile *config.Profile, opts ResolveCredentialOptions) (*config.Profile, Credential, error) { + if profile == nil { + return nil, Credential{}, apperr.New("fs.missing_profile", "config", 2, "active profile is required") + } + id := strings.TrimSpace(opts.FileSystemID) + if opts.FileSystemIDExplicit && id == "" { + return nil, Credential{}, apperr.New("fs.empty_file_system_id", "usage", 2, "--file-system-id cannot be empty") + } + if id == "" { + value, err := fsEnvValue(opts.Env, "TI_FS_FILE_SYSTEM_ID") + if err != nil { + return nil, Credential{}, err + } + id = strings.TrimSpace(value) + } + token := strings.TrimSpace(opts.Token) + if opts.TokenExplicit && token == "" { + return nil, Credential{}, apperr.New("fs.empty_token", "usage", 2, "--fs-token cannot be empty") + } + if token == "" { + value, err := fsEnvValue(opts.Env, "TI_FS_TOKEN") + if err != nil { + return nil, Credential{}, err + } + token = strings.TrimSpace(value) + } + explicitToken := token != "" + if explicitToken { + tokenID, err := FileSystemIDFromToken(token) + if err != nil { + return nil, Credential{}, err + } + if id == "" { + id = tokenID + } else if id != tokenID { + return nil, Credential{}, apperr.New("fs.token_file_system_mismatch", "authentication", 3, fmt.Sprintf("FS token belongs to file system %q, not %q", tokenID, id)) + } + } + if id == "" { + return nil, Credential{}, missingFileSystemID() + } + var credential Credential + found := false + if !opts.DryRun || !explicitToken { + stored, err := GetCredential(homeDir, profile.Name, id) + if err == nil { + credential = stored + found = true + } else if apperr.CodeFor(err) != "fs.credential_not_found" { + return nil, Credential{}, err + } + } + if opts.DryRun && !found { + legacy := legacyResource(profile) + if legacy.TenantID == id && legacy.APIKey != "" { + credential = Credential{FileSystemID: id, RegionCode: legacy.RegionCode, HasLocalToken: true, APIKey: legacy.APIKey} + found = true + } else if resources, err := List(homeDir, profile.Name); err == nil { + for _, resource := range resources { + if resource.TenantID == id { + credential = Credential{FileSystemID: id, RegionCode: resource.RegionCode, HasLocalToken: true, APIKey: resource.APIKey} + found = true + break + } + } + } + } + if token == "" && found { + token = credential.APIKey + } + if opts.TokenRequired && token == "" { + return nil, Credential{}, apperr.New( + "auth.missing_fs_api_key", + "authentication", + 3, + fmt.Sprintf("authentication required: no local FS token is stored for file system %q; pass --fs-token, set TI_FS_TOKEN, or import a known token with `ti fs import-file-system-token`. Token regeneration is not available yet", id), + ) + } + placementCode := strings.TrimSpace(opts.RegionOverride) + if placementCode == "" && found { + placementCode = credential.RegionCode + } + if placementCode == "" { + placementCode = strings.TrimSpace(profile.PlacementRegionCode) + } + if placementCode == "" { + return nil, Credential{}, apperr.New("fs.missing_region", "config", 2, "ti fs region is required; pass --region, set TI_REGION_CODE, or use locally stored credentials with region_code") + } + placement, err := region.ParsePlacementCode(placementCode) + if err != nil { + return nil, Credential{}, apperr.Wrap("config.invalid_region", "config", 2, err.Error(), err) + } + if found && credential.RegionCode != placement.Code { + return nil, Credential{}, apperr.New("fs.credential_region_mismatch", "config", 2, fmt.Sprintf("file system %q credentials are for %s, not %s", id, credential.RegionCode, placement.Code)) + } + credential.FileSystemID = id + credential.RegionCode = placement.Code + credential.APIKey = token + credential.HasLocalToken = token != "" + selected := *profile + selected.FSResourceName = id + selected.FSTenantID = id + selected.FSPlacementRegionCode = placement.Code + selected.FSCloudProvider = placement.Provider + selected.FSRegionCode = placement.NativeCode + selected.FSAPIKey = token + return &selected, credential, nil +} + +func FileSystemIDFromToken(raw string) (string, error) { + raw = strings.TrimSpace(raw) + var wrapped string + switch { + case strings.HasPrefix(raw, "drive9_"): + wrapped = strings.TrimPrefix(raw, "drive9_") + case strings.HasPrefix(raw, "dat9_"): + wrapped = strings.TrimPrefix(raw, "dat9_") + default: + return "", apperr.New("fs.invalid_token", "authentication", 3, "invalid FS token format") + } + jwtBytes, err := base64.RawURLEncoding.DecodeString(wrapped) + if err != nil { + return "", apperr.Wrap("fs.invalid_token", "authentication", 3, "invalid FS token wrapper", err) + } + parts := strings.Split(string(jwtBytes), ".") + if len(parts) != 3 { + return "", apperr.New("fs.invalid_token", "authentication", 3, "invalid FS token JWT structure") + } + payload, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + return "", apperr.Wrap("fs.invalid_token", "authentication", 3, "invalid FS token JWT payload", err) + } + var claims struct { + TenantID string `json:"tenant_id"` + } + if err := json.Unmarshal(payload, &claims); err != nil { + return "", apperr.Wrap("fs.invalid_token", "authentication", 3, "invalid FS token JWT claims", err) + } + if strings.TrimSpace(claims.TenantID) == "" { + return "", apperr.New("fs.invalid_token", "authentication", 3, "invalid FS token JWT claims: tenant_id is missing") + } + id, err := ValidateFileSystemID(claims.TenantID) + if err != nil { + return "", apperr.Wrap("fs.invalid_token", "authentication", 3, "invalid FS token JWT tenant_id", err) + } + return id, nil +} + +func ValidateFileSystemID(value string) (string, error) { + id := strings.TrimSpace(value) + if id == "" { + return "", apperr.New("fs.missing_file_system_id", "usage", 2, "--file-system-id is required unless an FS token is supplied") + } + if len(id) > 128 || strings.ContainsAny(id, "/\\") { + return "", apperr.New("fs.invalid_file_system_id", "usage", 2, "file system ID must be 1-128 characters and must not contain path separators") + } + for _, r := range id { + if r < 0x21 || r == 0x7f { + return "", apperr.New("fs.invalid_file_system_id", "usage", 2, "file system ID must not contain whitespace or control characters") + } + } + return id, nil +} + +func MigrateNameRegistry(homeDir string, profile *config.Profile) error { + if profile == nil { + return nil + } + migrationMu.Lock() + defer migrationMu.Unlock() + if err := MigrateLegacy(homeDir, profile); err != nil { + return err + } + resources, err := List(homeDir, profile.Name) + if err != nil { + return err + } + candidates := make(map[string]Credential, len(resources)) + for _, resource := range resources { + candidate := Credential{FileSystemID: resource.TenantID, RegionCode: resource.RegionCode, APIKey: resource.APIKey} + if previous, ok := candidates[resource.TenantID]; ok && (previous.RegionCode != candidate.RegionCode || previous.APIKey != candidate.APIKey) { + return credentialError("fs.credential_migration_conflict", profile.Name, resource.TenantID, "legacy names contain conflicting credentials for the same file system ID") + } + candidates[resource.TenantID] = candidate + } + migrated, err := loadMigrationState(homeDir, profile.Name) + if err != nil { + return err + } + pending := make(map[string]Credential, len(candidates)) + for id, candidate := range candidates { + if migrated[id] { + continue + } + pending[id] = candidate + } + for id, candidate := range pending { + if existing, err := GetCredential(homeDir, profile.Name, id); err == nil { + if existing.RegionCode != candidate.RegionCode || existing.APIKey != candidate.APIKey { + return credentialError("fs.credential_migration_conflict", profile.Name, id, "legacy credentials conflict with the ID-keyed credential") + } + } else if apperr.CodeFor(err) != "fs.credential_not_found" { + return err + } + } + for id, candidate := range pending { + if _, err := StoreCredential(homeDir, profile, id, candidate.RegionCode, candidate.APIKey, false); err != nil { + if apperr.CodeFor(err) == "fs.credential_import_conflict" { + return credentialError("fs.credential_migration_conflict", profile.Name, id, "legacy credentials conflict with the ID-keyed credential") + } + return err + } + } + if len(pending) > 0 { + for id := range pending { + migrated[id] = true + } + if err := writeMigrationState(homeDir, profile.Name, migrated); err != nil { + return err + } + } + return nil +} + +func loadMigrationState(homeDir, profileName string) (map[string]bool, error) { + path := filepath.Join(credentialProfileDir(homeDir, profileName), migrationStateFileName) + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return map[string]bool{}, nil + } + if err != nil { + return nil, err + } + var state migrationState + if err := toml.Unmarshal(data, &state); err != nil || state.SchemaVersion != migrationStateSchema { + return nil, credentialError("fs.credential_migration_state_invalid", profileName, "", "cannot parse the legacy registry migration state") + } + ids := make(map[string]bool, len(state.FileSystemIDs)) + for _, value := range state.FileSystemIDs { + id, err := ValidateFileSystemID(value) + if err != nil { + return nil, credentialError("fs.credential_migration_state_invalid", profileName, "", "legacy registry migration state contains an invalid file system ID") + } + ids[id] = true + } + return ids, nil +} + +func writeMigrationState(homeDir, profileName string, migrated map[string]bool) error { + ids := make([]string, 0, len(migrated)) + for id, done := range migrated { + if done { + ids = append(ids, id) + } + } + sort.Strings(ids) + path := filepath.Join(credentialProfileDir(homeDir, profileName), migrationStateFileName) + if err := writeTOML(path, migrationState{SchemaVersion: migrationStateSchema, FileSystemIDs: ids}, 0o600); err != nil { + return credentialError("fs.credential_migration_failed", profileName, "", "cannot write the legacy registry migration state") + } + return nil +} + +func missingFileSystemID() error { + return apperr.New("fs.missing_file_system_id", "usage", 2, "file system ID is required; pass --file-system-id, set TI_FS_FILE_SYSTEM_ID, or supply an FS token") +} + +func credentialDir(homeDir, profileName, fileSystemID string) (string, error) { + id, err := ValidateFileSystemID(fileSystemID) + if err != nil { + return "", err + } + return filepath.Join(credentialProfileDir(homeDir, profileName), encodeKey(id)), nil +} + +func credentialProfileDir(homeDir, profileName string) string { + return filepath.Join(homeDir, store.TIDirName, credentialsDirName, encodeKey(normalizedProfile(profileName))) +} + +func ensureCredentialDirs(homeDir, profileName, fileSystemID string) error { + root := filepath.Join(homeDir, store.TIDirName, credentialsDirName) + profilePath := credentialProfileDir(homeDir, profileName) + credentialPath, err := credentialDir(homeDir, profileName, fileSystemID) + if err != nil { + return err + } + for _, path := range []string{root, profilePath, credentialPath} { + if err := os.MkdirAll(path, 0o700); err != nil { + return err + } + if err := os.Chmod(path, 0o700); err != nil { + return err + } + } + return nil +} + +func credentialError(code, profileName, fileSystemID, detail string) error { + message := fmt.Sprintf("%s for profile %q", detail, normalizedProfile(profileName)) + if strings.TrimSpace(fileSystemID) != "" { + message += fmt.Sprintf(" and file system ID %q", strings.TrimSpace(fileSystemID)) + } + return apperr.New(code, "config", 2, message) +} diff --git a/internal/fs/fscred/credential_test.go b/internal/fs/fscred/credential_test.go new file mode 100644 index 0000000..d5e06f4 --- /dev/null +++ b/internal/fs/fscred/credential_test.go @@ -0,0 +1,270 @@ +package fscred + +import ( + "encoding/base64" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/tidbcloud/ti-cli/internal/apperr" + "github.com/tidbcloud/ti-cli/internal/config" +) + +func TestCredentialStoreAndResolveByID(t *testing.T) { + home := t.TempDir() + profile := credentialTestProfile() + stored, err := StoreCredential(home, profile, "tenant-1", "aws-us-east-1", wrappedToken(t, "tenant-1"), false) + if err != nil { + t.Fatal(err) + } + if stored.FileSystemID != "tenant-1" || !stored.HasLocalToken { + t.Fatalf("stored credential = %#v", stored) + } + paths, err := CredentialPath(home, profile.Name, "tenant-1") + if err != nil { + t.Fatal(err) + } + info, err := os.Stat(paths.Credentials) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o600 { + t.Fatalf("credential mode = %#o", info.Mode().Perm()) + } + for _, dir := range []string{filepath.Dir(paths.Credentials), filepath.Dir(filepath.Dir(paths.Credentials))} { + info, err := os.Stat(dir) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o700 { + t.Fatalf("credential directory %s mode = %#o", dir, info.Mode().Perm()) + } + } + selected, credential, err := ResolveCredential(home, profile, ResolveCredentialOptions{FileSystemID: "tenant-1", FileSystemIDExplicit: true, TokenRequired: true}) + if err != nil { + t.Fatal(err) + } + if selected.FSTenantID != "tenant-1" || selected.FSAPIKey == "" || credential.RegionCode != "aws-us-east-1" { + t.Fatalf("selected=%#v credential=%#v", selected, credential) + } +} + +func TestResolveCredentialDerivesIDFromExplicitToken(t *testing.T) { + profile := credentialTestProfile() + token := wrappedToken(t, "tenant-token") + selected, credential, err := ResolveCredential(t.TempDir(), profile, ResolveCredentialOptions{ + Token: token, TokenExplicit: true, RegionOverride: "aws-us-east-1", TokenRequired: true, + }) + if err != nil { + t.Fatal(err) + } + if selected.FSTenantID != "tenant-token" || credential.FileSystemID != "tenant-token" || credential.APIKey != token { + t.Fatalf("selected=%#v credential=%#v", selected, credential) + } + _, _, err = ResolveCredential(t.TempDir(), profile, ResolveCredentialOptions{ + FileSystemID: "tenant-other", FileSystemIDExplicit: true, Token: token, TokenExplicit: true, RegionOverride: "aws-us-east-1", TokenRequired: true, + }) + if apperr.CodeFor(err) != "fs.token_file_system_mismatch" { + t.Fatalf("mismatch error = %v", err) + } +} + +func TestResolveCredentialReportsMissingLocalTokenForKnownID(t *testing.T) { + _, _, err := ResolveCredential(t.TempDir(), credentialTestProfile(), ResolveCredentialOptions{ + FileSystemID: "tenant-without-token", FileSystemIDExplicit: true, TokenRequired: true, + }) + if apperr.CodeFor(err) != "auth.missing_fs_api_key" { + t.Fatalf("missing token error = %v", err) + } + if !strings.Contains(err.Error(), "import-file-system-token") || !strings.Contains(err.Error(), "not available yet") { + t.Fatalf("missing token error is not actionable: %v", err) + } +} + +func TestFileSystemIDFromTokenRejectsMalformedInputs(t *testing.T) { + wrapJWT := func(jwt string) string { + return "drive9_" + base64.RawURLEncoding.EncodeToString([]byte(jwt)) + } + encoded := func(value string) string { + return base64.RawURLEncoding.EncodeToString([]byte(value)) + } + for _, tc := range []struct { + name string + token string + }{ + {name: "empty", token: ""}, + {name: "unknown wrapper", token: "opaque"}, + {name: "invalid wrapper encoding", token: "drive9_%%%"}, + {name: "invalid JWT structure", token: wrapJWT("only.two")}, + {name: "invalid claims JSON", token: wrapJWT("header." + encoded("{") + ".signature")}, + {name: "missing tenant ID", token: wrapJWT("header." + encoded(`{"token_version":1}`) + ".signature")}, + {name: "invalid tenant ID", token: wrapJWT("header." + encoded(`{"tenant_id":"bad id"}`) + ".signature")}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, err := FileSystemIDFromToken(tc.token); apperr.CodeFor(err) != "fs.invalid_token" { + t.Fatalf("error = %v, want fs.invalid_token", err) + } + }) + } +} + +func TestCredentialConflictAndExplicitReplace(t *testing.T) { + home := t.TempDir() + profile := credentialTestProfile() + first := wrappedToken(t, "tenant-1") + if _, err := StoreCredential(home, profile, "tenant-1", "aws-us-east-1", first, false); err != nil { + t.Fatal(err) + } + second := first + "changed" + if _, err := StoreCredential(home, profile, "tenant-1", "aws-us-east-1", second, false); apperr.CodeFor(err) != "fs.credential_import_conflict" { + t.Fatalf("conflict error = %v", err) + } + if _, err := StoreCredential(home, profile, "tenant-1", "aws-us-east-1", second, true); err != nil { + t.Fatal(err) + } + got, err := GetCredential(home, profile.Name, "tenant-1") + if err != nil || got.APIKey != second { + t.Fatalf("credential=%#v err=%v", got, err) + } +} + +func TestMigrateNameRegistryPreservesLegacySource(t *testing.T) { + home := t.TempDir() + profile := credentialTestProfile() + token := wrappedToken(t, "tenant-legacy") + if err := Store(home, profile, "workspace", "tenant-legacy", "aws", "aws-us-east-1", token); err != nil { + t.Fatal(err) + } + if err := MigrateNameRegistry(home, profile); err != nil { + t.Fatal(err) + } + if _, err := Get(home, profile.Name, "workspace"); err != nil { + t.Fatalf("legacy source removed: %v", err) + } + credential, err := GetCredential(home, profile.Name, "tenant-legacy") + if err != nil || credential.APIKey != token { + t.Fatalf("credential=%#v err=%v", credential, err) + } +} + +func TestMigrateNameRegistryPreflightsDuplicateIDConflicts(t *testing.T) { + home := t.TempDir() + profile := credentialTestProfile() + if err := Store(home, profile, "first", "tenant-shared", "aws", "aws-us-east-1", wrappedToken(t, "tenant-shared")); err != nil { + t.Fatal(err) + } + if err := Store(home, profile, "second", "tenant-shared", "aws", "aws-us-east-1", wrappedToken(t, "tenant-shared")+"-different"); err != nil { + t.Fatal(err) + } + err := MigrateNameRegistry(home, profile) + if apperr.CodeFor(err) != "fs.credential_migration_conflict" { + t.Fatalf("migration error = %v", err) + } + if _, err := GetCredential(home, profile.Name, "tenant-shared"); apperr.CodeFor(err) != "fs.credential_not_found" { + t.Fatalf("conflicting migration wrote a destination credential: %v", err) + } + if _, err := Get(home, profile.Name, "first"); err != nil { + t.Fatalf("first legacy source changed: %v", err) + } + if _, err := Get(home, profile.Name, "second"); err != nil { + t.Fatalf("second legacy source changed: %v", err) + } +} + +func TestMigrateNameRegistryMultipleResourcesAliasesAndIdempotency(t *testing.T) { + home := t.TempDir() + profile := credentialTestProfile() + sharedToken := wrappedToken(t, "tenant-shared") + for _, resource := range []struct { + name, id, token string + }{ + {name: "workspace", id: "tenant-shared", token: sharedToken}, + {name: "workspace-alias", id: "tenant-shared", token: sharedToken}, + {name: "scratch", id: "tenant-scratch", token: wrappedToken(t, "tenant-scratch")}, + } { + if err := Store(home, profile, resource.name, resource.id, "aws", "aws-us-east-1", resource.token); err != nil { + t.Fatal(err) + } + } + for i := 0; i < 2; i++ { + if err := MigrateNameRegistry(home, profile); err != nil { + t.Fatalf("migration pass %d: %v", i+1, err) + } + } + credentials, err := ListCredentials(home, profile.Name) + if err != nil { + t.Fatal(err) + } + if len(credentials) != 2 { + t.Fatalf("migrated credentials = %#v, want two unique IDs", credentials) + } + for _, name := range []string{"workspace", "workspace-alias", "scratch"} { + if _, err := Get(home, profile.Name, name); err != nil { + t.Fatalf("legacy source %q was changed: %v", name, err) + } + } +} + +func TestMigrateNameRegistryDoesNotRestoreDeletedCredential(t *testing.T) { + home := t.TempDir() + profile := credentialTestProfile() + if err := Store(home, profile, "workspace", "tenant-deleted", "aws", "aws-us-east-1", wrappedToken(t, "tenant-deleted")); err != nil { + t.Fatal(err) + } + if err := MigrateNameRegistry(home, profile); err != nil { + t.Fatal(err) + } + if removed, err := DeleteCredential(home, profile.Name, "tenant-deleted"); err != nil || !removed { + t.Fatalf("delete migrated credential: removed=%t err=%v", removed, err) + } + if err := MigrateNameRegistry(home, profile); err != nil { + t.Fatal(err) + } + if _, err := GetCredential(home, profile.Name, "tenant-deleted"); apperr.CodeFor(err) != "fs.credential_not_found" { + t.Fatalf("completed legacy migration restored a deleted credential: %v", err) + } + if _, err := Get(home, profile.Name, "workspace"); err != nil { + t.Fatalf("legacy rollback source was removed: %v", err) + } +} + +func TestMigrateNameRegistryPreflightsDestinationConflictsBeforeAnyWrite(t *testing.T) { + home := t.TempDir() + profile := credentialTestProfile() + if err := Store(home, profile, "first", "tenant-first", "aws", "aws-us-east-1", wrappedToken(t, "tenant-first")); err != nil { + t.Fatal(err) + } + legacyConflictToken := wrappedToken(t, "tenant-conflict") + if err := Store(home, profile, "conflict", "tenant-conflict", "aws", "aws-us-east-1", legacyConflictToken); err != nil { + t.Fatal(err) + } + if _, err := StoreCredential(home, profile, "tenant-conflict", "aws-us-east-1", legacyConflictToken+"-different", false); err != nil { + t.Fatal(err) + } + if err := MigrateNameRegistry(home, profile); apperr.CodeFor(err) != "fs.credential_migration_conflict" { + t.Fatalf("migration error = %v", err) + } + if _, err := GetCredential(home, profile.Name, "tenant-first"); apperr.CodeFor(err) != "fs.credential_not_found" { + t.Fatalf("preflight conflict allowed a partial migration: %v", err) + } +} + +func wrappedToken(t *testing.T, tenantID string) string { + t.Helper() + header := base64.RawURLEncoding.EncodeToString([]byte(`{"alg":"HS256","typ":"JWT"}`)) + payloadBytes, err := json.Marshal(map[string]any{"tenant_id": tenantID, "token_version": 1, "iat": 1}) + if err != nil { + t.Fatal(err) + } + payload := base64.RawURLEncoding.EncodeToString(payloadBytes) + jwt := header + "." + payload + ".signature" + return "drive9_" + base64.RawURLEncoding.EncodeToString([]byte(jwt)) +} + +func credentialTestProfile() *config.Profile { + return &config.Profile{ + Name: "stage", PlacementRegionCode: "aws-us-east-1", CloudProvider: "aws", RegionCode: "us-east-1", + } +} diff --git a/internal/fs/mount.go b/internal/fs/mount.go index 85d3085..d94e786 100644 --- a/internal/fs/mount.go +++ b/internal/fs/mount.go @@ -77,7 +77,7 @@ type DrainFileSystemOptions struct { type MountResult struct { Status string `json:"status"` Profile string `json:"profile"` - FileSystemName string `json:"file_system_name"` + FileSystemName string `json:"file_system_id"` MountPath string `json:"mount_path"` RemotePath string `json:"remote_path"` Driver string `json:"driver"` @@ -236,7 +236,7 @@ func (s Service) DryRunDrainFileSystem(ctx context.Context, commandPath string, Path: opts.MountPath, }, dryrun.Check{Name: "mount_locator", Status: "passed", Message: opts.MountPath}, - dryrun.Check{Name: "file_system_name", Status: "passed", Message: profile.FSResourceName}, + dryrun.Check{Name: "file_system_id", Status: "passed", Message: profile.FSTenantID}, dryrun.Check{Name: "region", Status: "passed", Message: profile.FSPlacementRegionCode}, ), nil } @@ -263,7 +263,7 @@ func (s Service) DryRunUnmountFileSystem(ctx context.Context, commandPath string Path: opts.MountPath, }, dryrun.Check{Name: "mount_locator", Status: "passed", Message: opts.MountPath}, - dryrun.Check{Name: "file_system_name", Status: "passed", Message: profile.FSResourceName}, + dryrun.Check{Name: "file_system_id", Status: "passed", Message: profile.FSTenantID}, dryrun.Check{Name: "region", Status: "passed", Message: profile.FSPlacementRegionCode}, ), nil } @@ -331,10 +331,10 @@ func (s Service) mountInputs(opts MountFileSystemOptions) (mountInputs, error) { fileSystemName = opts.Profile.FSResourceName } if fileSystemName == "" { - return mountInputs{}, apperr.New("fs.missing_file_system_name", "usage", 2, "--file-system-name is required or fs_resource_name must exist in the active profile") + return mountInputs{}, apperr.New("fs.missing_file_system_id", "usage", 2, "--file-system-id is required unless an FS token identifies the file system") } if opts.Profile.FSResourceName != "" && opts.Profile.FSResourceName != fileSystemName { - return mountInputs{}, resourceMismatch(opts.Profile.FSResourceName, fileSystemName) + return mountInputs{}, apperr.New("fs.file_system_id_mismatch", "usage", 2, fmt.Sprintf("resolved file system ID %q does not match %q", opts.Profile.FSResourceName, fileSystemName)) } mountPath, err := mountstate.CanonicalMountPath(opts.MountPath) if err != nil { @@ -568,7 +568,7 @@ func (s Service) mountBackground(ctx context.Context, inputs mountInputs, remote args := []string{ "--profile", inputs.profile.Name, "fs", "mount-file-system", - "--file-system-name", inputs.fileSystemName, + "--file-system-id", inputs.fileSystemName, "--mount-path", inputs.mountPath, "--remote-path", inputs.remotePath, "--driver", inputs.driver.Name(), diff --git a/ref/drive9 b/ref/drive9 index f0b13af..5f6779c 160000 --- a/ref/drive9 +++ b/ref/drive9 @@ -1 +1 @@ -Subproject commit f0b13af3176e8fc9c9e5a9d5fd4fcbb1855c73cc +Subproject commit 5f6779c4549f925845176a20e011c2021ecf680b diff --git a/ref/fs b/ref/fs new file mode 160000 index 0000000..4e467e3 --- /dev/null +++ b/ref/fs @@ -0,0 +1 @@ +Subproject commit 4e467e300136a9fefad5cc7728dab716127efbec diff --git a/scripts/install.ps1 b/scripts/install.ps1 index 6443437..0b1e865 100644 --- a/scripts/install.ps1 +++ b/scripts/install.ps1 @@ -127,11 +127,11 @@ function Print-NextSteps { Write-Output " ti organization list-projects --output text" Write-Output "" Write-Output " 4. Create or check ti fs" - Write-Output " ti fs create-file-system --file-system-name workspace" + Write-Output ' $env:TI_FS_FILE_SYSTEM_ID = ti fs create-file-system --query file_system_id --output text' Write-Output " ti fs check-file-system --output text" Write-Output "" Write-Output " 5. Mount ti fs when FUSE is available" - Write-Output " ti fs mount-file-system --file-system-name workspace --mount-path ./workspace" + Write-Output ' ti fs mount-file-system --file-system-id $env:TI_FS_FILE_SYSTEM_ID --mount-path ./workspace' Write-Output "" Write-Output " Docs: https://github.com/tidbcloud/ti-cli" } diff --git a/scripts/install.sh b/scripts/install.sh index 06732ab..d3e43ef 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -314,11 +314,11 @@ print_next_steps() { printf " ${DIM}\$${RESET} ti organization list-projects --output text\n" printf "\n" printf " ${BOLD}4.${RESET} Create or check ti fs\n" - printf " ${DIM}\$${RESET} ti fs create-file-system --file-system-name workspace\n" + printf " ${DIM}\$${RESET} export TI_FS_FILE_SYSTEM_ID=\$(ti fs create-file-system --query file_system_id --output text)\n" printf " ${DIM}\$${RESET} ti fs check-file-system --output text\n" printf "\n" printf " ${BOLD}5.${RESET} Mount ti fs when FUSE is available\n" - printf " ${DIM}\$${RESET} ti fs mount-file-system --file-system-name workspace --mount-path ./workspace\n" + printf " ${DIM}\$${RESET} ti fs mount-file-system --file-system-id \"\$TI_FS_FILE_SYSTEM_ID\" --mount-path ./workspace\n" printf "\n" printf " Docs: ${DIM}https://github.com/tidbcloud/ti-cli${RESET}\n" }