Make --profile take precedence over auth environment variables

NEXT_CHANGELOG.md

@@ -8,6 +8,7 @@
 * `workspace export-dir` no longer aborts when a workspace object's name is not a legal local filename (e.g. a notebook named `New Notebook 2026-05-04 13:54:24` whose `:` is illegal on Windows). Such files are now exported under a sanitized name with a warning and the export completes ([#5171](https://github.com/databricks/cli/issues/5171)).
 * `ssh connect` now opens an interactive `bash` login shell by default instead of the compute image's default `/bin/sh`, falling back gracefully when `bash` is unavailable. Passing an explicit remote command (`-- <cmd>`) is unaffected ([#5687](https://github.com/databricks/cli/pull/5687)).
 * `ssh connect` interactive sessions now start in the user's workspace home folder (`/Workspace/Users/<email>`) instead of the OS home directory, falling back to the OS home when that folder is unavailable ([#5688](https://github.com/databricks/cli/pull/5688)).
+* An explicitly selected profile (the `--profile` flag or a bundle's `workspace.profile`) now takes precedence over authentication environment variables (`DATABRICKS_HOST`, `DATABRICKS_TOKEN`, etc.) for the fields it sets, instead of being silently shadowed by them. Environment variables still fill auth fields the profile leaves empty (e.g. a host-only profile combined with `DATABRICKS_TOKEN`). A profile picked up from `DATABRICKS_CONFIG_PROFILE` keeps the SDK's default env-first precedence ([#5096](https://github.com/databricks/cli/issues/5096)).
 
 ### Bundles
 * Add documentation for the common bundle resource fields `permissions`, `lifecycle`, and `grants` in the JSON schema, so they surface in editor completions and the docs.

acceptance/cmd/api/profile-overrides-env/output.txt

@@ -0,0 +1,44 @@
+
+=== api --profile overrides auth env vars (#5096)
+
+>>> [CLI] api get /api/2.0/clusters/list --profile my-workspace
+{}
+
+>>> print_requests.py --get //api/2.0/clusters/list
+{
+  "headers": {
+    "Authorization": [
+      "Bearer [DATABRICKS_TOKEN]"
+    ],
+    "User-Agent": [
+      "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/api_get cmd-exec-id/[UUID] interactive/none auth/pat"
+    ],
+    "X-Databricks-Workspace-Id": [
+      "[NUMID]"
+    ]
+  },
+  "method": "GET",
+  "path": "/api/2.0/clusters/list"
+}
+
+=== api host-only --profile fills the token from the environment (#5096)
+
+>>> [CLI] api get /api/2.0/clusters/list --profile host-only
+{}
+
+>>> print_requests.py --get //api/2.0/clusters/list
+{
+  "headers": {
+    "Authorization": [
+      "Bearer [DATABRICKS_TOKEN]"
+    ],
+    "User-Agent": [
+      "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/api_get cmd-exec-id/[UUID] interactive/none auth/pat"
+    ],
+    "X-Databricks-Workspace-Id": [
+      "[NUMID]"
+    ]
+  },
+  "method": "GET",
+  "path": "/api/2.0/clusters/list"
+}

acceptance/cmd/api/profile-overrides-env/script

@@ -0,0 +1,29 @@
+sethome "./home"
+
+# One profile with full credentials, one host-only; both point at the test
+# server while the auth env vars below point elsewhere.
+cat > "./home/.databrickscfg" <<EOF
+[my-workspace]
+host  = $DATABRICKS_HOST
+token = $DATABRICKS_TOKEN
+
+[host-only]
+host  = $DATABRICKS_HOST
+EOF
+
+# direnv-style auth env vars for a different workspace; before #5096 these
+# shadowed the profile selected with --profile.
+real_token=$DATABRICKS_TOKEN
+export DATABRICKS_HOST=https://dev.cloud.databricks.test
+export DATABRICKS_TOKEN=dev-token
+
+title "api --profile overrides auth env vars (#5096)\n"
+MSYS_NO_PATHCONV=1 trace $CLI api get /api/2.0/clusters/list --profile my-workspace
+trace print_requests.py --get //api/2.0/clusters/list
+
+# Host-only profile: the profile wins for the host, but env fills the token it
+# omits (#5096).
+export DATABRICKS_TOKEN=$real_token
+title "api host-only --profile fills the token from the environment (#5096)\n"
+MSYS_NO_PATHCONV=1 trace $CLI api get /api/2.0/clusters/list --profile host-only
+trace print_requests.py --get //api/2.0/clusters/list

acceptance/cmd/api/profile-overrides-env/test.toml

@@ -0,0 +1,3 @@
+Ignore = [
+    "home",
+]

acceptance/cmd/auth/describe/profile-overrides-env/output.txt

@@ -0,0 +1,36 @@
+
+=== Describe with --profile overrides auth env vars (#5096)
+
+>>> [CLI] auth describe --profile my-workspace
+Host: [DATABRICKS_URL]
+User: [USERNAME]
+Authenticated with: pat
+-----
+Current configuration:
+  ✓ host: [DATABRICKS_URL] (from [TEST_TMP_DIR]/home/.databrickscfg config file)
+  ✓ workspace_id: [NUMID]
+  ✓ token: ******** (from [TEST_TMP_DIR]/home/.databrickscfg config file)
+  ✓ profile: my-workspace (from --profile flag)
+  ✓ databricks_cli_path: [CLI]
+  ✓ auth_type: pat
+  ✓ rate_limit: [NUMID] (from DATABRICKS_RATE_LIMIT environment variable)
+  ✓ cloud: AWS
+  ✓ discovery_url: [DATABRICKS_URL]/oidc/.well-known/oauth-authorization-server
+
+=== Describe with a host-only --profile fills the token from the environment (#5096)
+
+>>> [CLI] auth describe --profile host-only
+Host: [DATABRICKS_URL]
+User: [USERNAME]
+Authenticated with: pat
+-----
+Current configuration:
+  ✓ host: [DATABRICKS_URL] (from [TEST_TMP_DIR]/home/.databrickscfg config file)
+  ✓ workspace_id: [NUMID]
+  ✓ token: ******** (from DATABRICKS_TOKEN environment variable)
+  ✓ profile: host-only (from --profile flag)
+  ✓ databricks_cli_path: [CLI]
+  ✓ auth_type: pat
+  ✓ rate_limit: [NUMID] (from DATABRICKS_RATE_LIMIT environment variable)
+  ✓ cloud: AWS
+  ✓ discovery_url: [DATABRICKS_URL]/oidc/.well-known/oauth-authorization-server

acceptance/cmd/auth/describe/profile-overrides-env/script

@@ -0,0 +1,26 @@
+sethome "./home"
+
+# A profile carries full credentials; a second profile carries only a host.
+cat > "./home/.databrickscfg" <<EOF
+[my-workspace]
+host  = $DATABRICKS_HOST
+token = $DATABRICKS_TOKEN
+
+[host-only]
+host  = $DATABRICKS_HOST
+EOF
+
+# direnv-style auth env vars for a different workspace; before #5096 these
+# shadowed the profile selected with --profile.
+real_token=$DATABRICKS_TOKEN
+export DATABRICKS_HOST=https://dev.cloud.databricks.test
+export DATABRICKS_TOKEN=dev-token
+
+title "Describe with --profile overrides auth env vars (#5096)\n"
+trace $CLI auth describe --profile my-workspace
+
+# Host-only profile: the profile wins for the host, but env fills the token it
+# omits (#5096).
+export DATABRICKS_TOKEN=$real_token
+title "Describe with a host-only --profile fills the token from the environment (#5096)\n"
+trace $CLI auth describe --profile host-only

acceptance/cmd/auth/describe/profile-overrides-env/test.toml

@@ -0,0 +1,3 @@
+Ignore = [
+    "home"
+]

bundle/config/workspace.go

@@ -176,9 +176,15 @@ func (w *Workspace) Client(ctx context.Context) (*databricks.WorkspaceClient, er
 
 	cfg := w.Config(ctx)
 
-	// If only the host is configured, we try and unambiguously match it to
-	// a profile in the user's databrickscfg file. Override the default loaders.
-	if w.Host != "" && w.Profile == "" {
+	switch {
+	case w.Profile != "":
+		// An explicit profile wins over auth env vars; see
+		// databrickscfg.ProfileAuthLoaders (#5096). ValidateConfigAndProfileHost
+		// below still checks the bundle host matches the profile host.
+		cfg.Loaders = databrickscfg.ProfileAuthLoaders
+	case w.Host != "":
+		// If only the host is configured, we try and unambiguously match it to
+		// a profile in the user's databrickscfg file. Override the default loaders.
 		cfg.Loaders = []config.Loader{
 			// Load auth creds from env vars
 			config.ConfigAttributes,

bundle/config/workspace_test.go

@@ -155,6 +155,80 @@ func TestWorkspaceClientNormalizesHostBeforeProfileResolution(t *testing.T) {
 	assert.Equal(t, "ws2", client.Config.Profile)
 }
 
+func TestWorkspaceClientProfileOverridesAuthEnv(t *testing.T) {
+	// An explicit profile (--profile or workspace.profile) must win over auth
+	// env vars, mirroring MustWorkspaceClient (#5096).
+	setupWorkspaceTest(t)
+
+	err := databrickscfg.SaveToProfile(t.Context(), &config.Config{
+		Profile: "tst",
+		Host:    "https://tst.cloud.databricks.test",
+		Token:   "tst-token",
+	})
+	require.NoError(t, err)
+
+	// direnv-style auth env vars pointing at a different (dev) workspace.
+	t.Setenv("DATABRICKS_HOST", "https://dev.cloud.databricks.test")
+	t.Setenv("DATABRICKS_TOKEN", "dev-token")
+
+	w := Workspace{Profile: "tst"}
+	client, err := w.Client(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, "tst", client.Config.Profile)
+	assert.Equal(t, "https://tst.cloud.databricks.test", client.Config.Host)
+	assert.Equal(t, "tst-token", client.Config.Token)
+}
+
+func TestWorkspaceClientProfileFillsAuthFromEnv(t *testing.T) {
+	// Host-only profile: the profile wins for the host, but env fills the token
+	// it omits (#5096).
+	setupWorkspaceTest(t)
+
+	err := databrickscfg.SaveToProfile(t.Context(), &config.Config{
+		Profile: "host-only",
+		Host:    "https://tst.cloud.databricks.test",
+	})
+	require.NoError(t, err)
+
+	t.Setenv("DATABRICKS_TOKEN", "env-token")
+
+	w := Workspace{Profile: "host-only"}
+	client, err := w.Client(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, "host-only", client.Config.Profile)
+	assert.Equal(t, "https://tst.cloud.databricks.test", client.Config.Host)
+	// The token is not in the profile, so it is filled from the environment.
+	assert.Equal(t, "env-token", client.Config.Token)
+}
+
+func TestWorkspaceClientHostAndProfileOverridesAuthEnv(t *testing.T) {
+	// A bundle that pins both workspace.host and workspace.profile: the profile
+	// still wins for auth over the env vars, the bundle host is honored, and
+	// ValidateConfigAndProfileHost passes because they agree (#5096).
+	setupWorkspaceTest(t)
+
+	err := databrickscfg.SaveToProfile(t.Context(), &config.Config{
+		Profile: "tst",
+		Host:    "https://tst.cloud.databricks.test",
+		Token:   "tst-token",
+	})
+	require.NoError(t, err)
+
+	// direnv-style auth env vars pointing at a different (dev) workspace.
+	t.Setenv("DATABRICKS_HOST", "https://dev.cloud.databricks.test")
+	t.Setenv("DATABRICKS_TOKEN", "dev-token")
+
+	w := Workspace{
+		Host:    "https://tst.cloud.databricks.test",
+		Profile: "tst",
+	}
+	client, err := w.Client(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, "tst", client.Config.Profile)
+	assert.Equal(t, "https://tst.cloud.databricks.test", client.Config.Host)
+	assert.Equal(t, "tst-token", client.Config.Token)
+}
+
 func TestWorkspaceConfigHTTPTimeout(t *testing.T) {
 	for _, tc := range []struct {
 		envVal string

cmd/api/api.go

@@ -85,7 +85,9 @@ func makeCommand(method string) *cobra.Command {
 			// also reads it, but setting cfg.Profile here keeps any error
 			// messages we render referring to the same name), 3.
 			// [__settings__].default_profile in the config file.
-			if profileFlag := cmd.Flag("profile"); profileFlag != nil {
+			profileFlag := cmd.Flag("profile")
+			hasProfileFlag := profileFlag != nil && profileFlag.Value.String() != ""
+			if hasProfileFlag {
 				cfg.Profile = profileFlag.Value.String()
 			}
 			if cfg.Profile == "" {
@@ -95,7 +97,14 @@ func makeCommand(method string) *cobra.Command {
 				cfg.Profile = databrickscfg.ResolveDefaultProfile(cmd.Context())
 			}
 
-			auth.NormalizeDatabricksConfigFromEnv(cmd.Context(), cfg)
+			if hasProfileFlag {
+				// An explicit --profile wins over auth env vars; see
+				// databrickscfg.ProfileAuthLoaders (#5096). Skip env host
+				// normalization too, since the host comes from the profile.
+				cfg.Loaders = databrickscfg.ProfileAuthLoaders
+			} else {
+				auth.NormalizeDatabricksConfigFromEnv(cmd.Context(), cfg)
+			}
 
 			api, err := client.New(cfg)
 			if err != nil {

cmd/root/auth.go

@@ -110,6 +110,18 @@ func profileFlagValue(cmd *cobra.Command) (string, bool) {
 	return value, value != ""
 }
 
+// applyProfileAuthPrecedence makes an explicitly selected --profile win over
+// auth env vars via databrickscfg.ProfileAuthLoaders (#5096). It also skips
+// NormalizeDatabricksConfigFromEnv, since the host comes from the profile rather
+// than DATABRICKS_HOST. Without a profile flag, env-first behavior is kept.
+func applyProfileAuthPrecedence(ctx context.Context, cfg *config.Config, hasProfileFlag bool) {
+	if hasProfileFlag {
+		cfg.Loaders = databrickscfg.ProfileAuthLoaders
+		return
+	}
+	auth.NormalizeDatabricksConfigFromEnv(ctx, cfg)
+}
+
 // Helper function to create an account client or prompt once if the given configuration is not valid.
 func accountClientOrPrompt(ctx context.Context, cfg *config.Config, allowPrompt bool) (*databricks.AccountClient, error) {
 	a, err := databricks.NewAccountClient((*databricks.Config)(cfg))
@@ -195,15 +207,15 @@ func MustAnyClient(cmd *cobra.Command, args []string) (bool, error) {
 
 func MustAccountClient(cmd *cobra.Command, args []string) error {
 	cfg := &config.Config{}
+	ctx := cmd.Context()
 
 	// The command-line profile flag takes precedence over DATABRICKS_CONFIG_PROFILE.
 	pr, hasProfileFlag := profileFlagValue(cmd)
 	if hasProfileFlag {
 		cfg.Profile = pr
 	}
+	applyProfileAuthPrecedence(ctx, cfg, hasProfileFlag)
 
-	ctx := cmd.Context()
-	auth.NormalizeDatabricksConfigFromEnv(ctx, cfg)
 	ctx = cmdctx.SetConfigUsed(ctx, cfg)
 	cmd.SetContext(ctx)
 
@@ -325,8 +337,7 @@ func MustWorkspaceClient(cmd *cobra.Command, args []string) error {
 	if hasProfileFlag {
 		cfg.Profile = profile
 	}
-
-	auth.NormalizeDatabricksConfigFromEnv(ctx, cfg)
+	applyProfileAuthPrecedence(ctx, cfg, hasProfileFlag)
 	resolveDefaultProfile(ctx, cfg)
 
 	_, isTargetFlagSet := targetFlagValue(cmd)