feat: Support Developer Edition connections (#2610)

* feat: Support AI Developer Edition connections through the Cloud SQL Auth Proxy

Cloud SQL AI Developer Edition instances allow connections through the SqlDataService. 
With this change and the corresponding chagnes in the Cloud SQL Go Connector, the Auth 
Proxy will allow users to connect to AIDE instances. Set the --sql-data flag to enable
this support.

* Internal changes to skip fuse on Kokoro tests.

* fix: force tcp4/tcp6 based on IP address type

This avoids binding to IPv6 wildcard [::] when 0.0.0.0 is requested, which can cause connection refused errors for IPv4 loopback (127.0.0.1) clients in environments where bindv6only is enabled.

* fix: add -buildvcs=false to go build in build.sh

* Update cloud-sql-go-connector to v1.22.0 and add --sql-data-endpoint alias

Update cloud-sql-go-connector dependency to v1.22.0 in go.mod.
Create --sql-data-endpoint as a hidden alias for --sqldata-api-endpoint,
supporting both CLI flags and environment variables.

TAG=agy
CONV=9333d162-5ca9-4d37-8cf4-47ce7a321b70

* Fix code review issues.

Author: hessjcg

.gitignore

@@ -13,3 +13,4 @@
 /logs/
 .tools
 test_results.txt
+test-results

Dockerfile.alpine

@@ -26,7 +26,7 @@ RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
     go build -ldflags "-X github.com/GoogleCloudPlatform/cloud-sql-proxy/v2/cmd.metadataString=container.alpine"
 
 # Final stage
-FROM alpine:3@sha256:4d889c14e7d5a73929ab00be2ef8ff22437e7cbc545931e52554a7b00e123d8b
+FROM alpine:3@sha256:79ff19e9084a00eece421b2523fb93e22d730e2c0e525905de047e848e56d95f
 
 LABEL org.opencontainers.image.source="https://github.com/GoogleCloudPlatform/cloud-sql-proxy"
 

Dockerfile.bookworm

@@ -26,7 +26,7 @@ RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
     go build -ldflags "-X github.com/GoogleCloudPlatform/cloud-sql-proxy/v2/cmd.metadataString=container.bookworm"
 
 # Final stage
-FROM gcr.io/cloud-marketplace-containers/google/debian12@sha256:d10f3a95f20cdf7e3d9d7cfbdcbdcc6ab26643714dab2d286e3030971d768188
+FROM gcr.io/cloud-marketplace-containers/google/debian12@sha256:8705219e4580f8c36633a84b491d35a66c3e788e81161d6b5f370daa948b9402
 
 LABEL org.opencontainers.image.source="https://github.com/GoogleCloudPlatform/cloud-sql-proxy"
 

build.sh

@@ -33,12 +33,25 @@ function clean() {
 
 ## build - Builds the project without running tests.
 function build() {
-   go build -o ./cloud-sql-proxy main.go
+   local metadata="${1:-}"
+   local ldflags=""
+   if [[ -n "$metadata" ]] ; then
+       ldflags="-X github.com/GoogleCloudPlatform/cloud-sql-proxy/v2/cmd.metadataString=$metadata"
+   fi
+   go build -buildvcs=false -ldflags "$ldflags" -o ./cloud-sql-proxy main.go
 }
 
 ## test - Runs local unit tests.
 function test() {
-  go test -v -race -cover -short ./...
+  get_golang_tool 'go-junit-report' 'jstemmer/go-junit-report' 'github.com/jstemmer/go-junit-report/v2'
+  mkdir -p test-results
+  local args=( "./..." )
+  if [[ "$#" -gt 0 ]] ; then
+    args=( "$@" )
+  fi
+  go test -v -race -cover -short "${args[@]}" -json \
+      | .tools/go-junit-report -iocopy -parser gojson -out test-results/unit.xml \
+      | jq -j 'select(.Output) | .Output '
 }
 
 ## e2e - Runs end-to-end integration tests.
@@ -53,7 +66,11 @@ function e2e() {
 # e2e_ci - Run end-to-end integration tests in the CI system.
 #   This assumes that the secrets in the env vars are already set.
 function e2e_ci() {
-  go test -race -v ./... | tee test_results.txt
+  get_golang_tool 'go-junit-report' 'jstemmer/go-junit-report' 'github.com/jstemmer/go-junit-report/v2'
+  mkdir -p test-results
+  go test -race -v ./...  -json \
+     | .tools/go-junit-report -iocopy -parser gojson -out test-results/e2e.xml \
+     | jq -j 'select(.Output) | .Output '
 }
 
 function get_golang_tool() {
@@ -227,16 +244,47 @@ function write_e2e_env(){
   done
 
   # Set IAM User env vars to the local gcloud user
-  echo "export MYSQL_IAM_USER='${local_user%%@*}'"
-  echo "export POSTGRES_USER_IAM='$local_user'"
+  echo "export MYSQL_IAM_USER='$(iam_user_mysql)'"
+  echo "export POSTGRES_USER_IAM='$(iam_user_pg)'"
   } > "$1"
 
 }
 
+function iam_user_pg() {
+  # Truncate the suffix `.iam.gserviceaccount.com` if it exists. Otherwise return the email.
+  local email
+  local pguser
+
+  email="$(iam_user_email)"
+  pguser="${email%%.iam.gserviceaccount.com}"
+  if [[ -n "$pguser" ]] ; then
+    echo "$pguser"
+  else
+    echo "$email"
+  fi
+
+}
+
+function iam_user_mysql() {
+  # Truncate the part after the @
+  local email
+  local mysqluser
+
+  email=$(iam_user_email)
+  mysqluser="${email%%@*}"
+  echo "$mysqluser"
+}
+
+function iam_user_email() {
+  gcloud auth list --format json | jq -r '.[] | select (.status == "ACTIVE") | .account'
+}
+
+
 ## build_image - Builds and pushes the proxy container image using local source.
-## Usage: ./build.sh build_image [image-url]
+## Usage: ./build.sh build_image [image-url] [metadata]
 function build_image() {
   local image_url="${1:-}"
+  local metadata="${2:-container}"
   local push_arg=""
 
   if [[ -n "$image_url" ]]; then
@@ -254,7 +302,7 @@ function build_image() {
   trap cleanup_build EXIT
 
   echo "Building binary locally..."
-  CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-X github.com/GoogleCloudPlatform/cloud-sql-proxy/v2/cmd.metadataString=container" -o cloud-sql-proxy
+  CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -buildvcs=false -ldflags "-X github.com/GoogleCloudPlatform/cloud-sql-proxy/v2/cmd.metadataString=$metadata" -o cloud-sql-proxy
 
   echo "Creating temporary Dockerfile..."
   cat > Dockerfile.local <<EOF

cmd/root.go

@@ -494,6 +494,12 @@ func NewCommand(opts ...Option) *Command {
 
 	// Flags that apply only to the root command
 	localFlags := rootCmd.Flags()
+	localFlags.SetNormalizeFunc(func(_ *pflag.FlagSet, name string) pflag.NormalizedName {
+		if name == "sql-data-endpoint" {
+			return pflag.NormalizedName("sqldata-api-endpoint")
+		}
+		return pflag.NormalizedName(name)
+	})
 	// Flags that apply to all sub-commands
 	globalFlags := rootCmd.PersistentFlags()
 
@@ -584,6 +590,9 @@ the cached copy has expired. Use this setting in environments where the
 CPU may be throttled and a background refresh cannot run reliably
 (e.g., Cloud Run)`,
 	)
+	localFlags.StringVar(&c.conf.SQLDataEndpoint, "sqldata-api-endpoint", "",
+		"Override the SQL Data API endpoint",
+	)
 
 	localFlags.BoolVar(&c.conf.RunConnectionTest, "run-connection-test", false, `Runs a connection test
 against all specified instances. If an instance is unreachable, the Proxy exits with a failure
@@ -606,6 +615,10 @@ only applicable to Unix sockets)`)
 		"(*) Connect to the private ip address for all instances")
 	localFlags.BoolVar(&c.conf.PSC, "psc", false,
 		"(*) Connect to the PSC endpoint for all instances")
+	localFlags.BoolVar(&c.conf.SQLDataEnabled, "sql-data", false,
+		"Enable SQL Data to tunnel through the Cloud SQL Admin API without"+
+			" needing network access to your public or private IP",
+	)
 
 	return c
 }
@@ -619,9 +632,14 @@ func loadConfig(c *Command, args []string, opts []Option) error {
 	c.Flags().VisitAll(func(f *pflag.Flag) {
 		// Override any unset flags with Viper values to use the pflags
 		// object as a single source of truth.
-		if !f.Changed && v.IsSet(f.Name) {
-			val := v.Get(f.Name)
-			_ = c.Flags().Set(f.Name, fmt.Sprintf("%v", val))
+		if !f.Changed {
+			if v.IsSet(f.Name) {
+				val := v.Get(f.Name)
+				_ = c.Flags().Set(f.Name, fmt.Sprintf("%v", val))
+			} else if f.Name == "sqldata-api-endpoint" && v.IsSet("sql-data-endpoint") {
+				val := v.Get("sql-data-endpoint")
+				_ = c.Flags().Set(f.Name, fmt.Sprintf("%v", val))
+			}
 		}
 	})
 
@@ -805,8 +823,18 @@ func parseConfig(cmd *Command, conf *proxy.Config, args []string) error {
 	}
 
 	// If more than one IP type is set, error.
-	if conf.PrivateIP && conf.PSC {
-		return newBadCommandError("cannot specify --private-ip and --psc flags at the same time")
+	var ipTypes int
+	if conf.PrivateIP {
+		ipTypes++
+	}
+	if conf.PSC {
+		ipTypes++
+	}
+	if conf.SQLDataEnabled {
+		ipTypes++
+	}
+	if ipTypes > 1 {
+		return newBadCommandError("cannot specify --private-ip, --psc, and --sql-data flags at the same time")
 	}
 
 	// If more than one auth method is set, error.
@@ -898,6 +926,7 @@ and re-try with just --auto-iam-authn`)
 			p, pok := q["port"]
 			u, uok := q["unix-socket"]
 			up, upok := q["unix-socket-path"]
+			sd, sdok := q["sql-data"]
 
 			if aok && uok {
 				return newBadCommandError("cannot specify both address and unix-socket query params")
@@ -955,6 +984,16 @@ and re-try with just --auto-iam-authn`)
 				}
 				ic.UnixSocketPath = up[0]
 			}
+			if sdok {
+				if len(sd) != 1 {
+					return newBadCommandError(fmt.Sprintf("sql-data query param should be only one value %q", a))
+				}
+				if sd[0] != "true" && sd[0] != "false" {
+					return newBadCommandError(fmt.Sprintf("sql-data query param should be \"true\" or \"false\" %q", a))
+				}
+				b := sd[0] == "true"
+				ic.SQLDataEnabled = &b
+			}
 
 			ic.IAMAuthN, err = parseBoolOpt(q, "auto-iam-authn")
 			if err != nil {

cmd/root_test.go

@@ -466,6 +466,36 @@ func TestNewCommandArguments(t *testing.T) {
 				RunConnectionTest: true,
 			}),
 		},
+		{
+			desc: "using the sql-data flag",
+			args: []string{"--sql-data", "proj:region:inst"},
+			want: withDefaults(&proxy.Config{
+				SQLDataEnabled: true,
+			}),
+		},
+		{
+			desc: "using the sqldata-api-endpoint flag",
+			args: []string{"--sqldata-api-endpoint", "https://test.googleapis.com", "proj:region:inst"},
+			want: withDefaults(&proxy.Config{
+				SQLDataEndpoint: "https://test.googleapis.com",
+			}),
+		},
+		{
+			desc: "using the sql-data-endpoint flag alias",
+			args: []string{"--sql-data-endpoint", "https://test.googleapis.com", "proj:region:inst"},
+			want: withDefaults(&proxy.Config{
+				SQLDataEndpoint: "https://test.googleapis.com",
+			}),
+		},
+		{
+			desc: "using the sql-data query param",
+			args: []string{"proj:region:inst?sql-data=true"},
+			want: withDefaults(&proxy.Config{
+				Instances: []proxy.InstanceConnConfig{{
+					SQLDataEnabled: pointer(true),
+				}},
+			}),
+		},
 	}
 
 	for _, tc := range tcs {
@@ -821,6 +851,30 @@ func TestNewCommandWithEnvironmentConfig(t *testing.T) {
 				AutoIP: true,
 			}),
 		},
+		{
+			desc:     "using the sql-data envvar",
+			envName:  "CSQL_PROXY_SQL_DATA",
+			envValue: "true",
+			want: withDefaults(&proxy.Config{
+				SQLDataEnabled: true,
+			}),
+		},
+		{
+			desc:     "using the sqldata-api-endpoint envvar",
+			envName:  "CSQL_PROXY_SQLDATA_API_ENDPOINT",
+			envValue: "https://test.googleapis.com",
+			want: withDefaults(&proxy.Config{
+				SQLDataEndpoint: "https://test.googleapis.com",
+			}),
+		},
+		{
+			desc:     "using the sql-data-endpoint envvar alias",
+			envName:  "CSQL_PROXY_SQL_DATA_ENDPOINT",
+			envValue: "https://test.googleapis.com",
+			want: withDefaults(&proxy.Config{
+				SQLDataEndpoint: "https://test.googleapis.com",
+			}),
+		},
 	}
 	for _, tc := range tcs {
 		t.Run(tc.desc, func(t *testing.T) {
@@ -1033,6 +1087,54 @@ func TestPSCQueryParams(t *testing.T) {
 	}
 }
 
+func TestSQLDataQueryParams(t *testing.T) {
+	tcs := []struct {
+		desc string
+		args []string
+		want *bool
+	}{
+		{
+			desc: "when the query string is absent",
+			args: []string{"proj:region:inst"},
+			want: nil,
+		},
+		{
+			desc: "when the query string is true",
+			args: []string{"proj:region:inst?sql-data=true"},
+			want: pointer(true),
+		},
+		{
+			desc: "when the query string is false",
+			args: []string{"proj:region:inst?sql-data=false"},
+			want: pointer(false),
+		},
+	}
+	for _, tc := range tcs {
+		t.Run(tc.desc, func(t *testing.T) {
+			c, err := invokeProxyCommand(tc.args)
+			if err != nil {
+				t.Fatalf("command.Execute: %v", err)
+			}
+			if tc.want == nil {
+				if len(c.conf.Instances) > 0 && c.conf.Instances[0].SQLDataEnabled != nil {
+					t.Fatalf("args = %v, want nil, got = %v", tc.args, *c.conf.Instances[0].SQLDataEnabled)
+				}
+				return
+			}
+			if len(c.conf.Instances) == 0 {
+				t.Fatal("expected at least one instance")
+			}
+			got := c.conf.Instances[0].SQLDataEnabled
+			if got == nil {
+				t.Fatalf("args = %v, want = %v, got = nil", tc.args, *tc.want)
+			}
+			if *got != *tc.want {
+				t.Errorf("args = %v, want = %v, got = %v", tc.args, *tc.want, *got)
+			}
+		})
+	}
+}
+
 func TestNewCommandWithErrors(t *testing.T) {
 	tcs := []struct {
 		desc string
@@ -1152,6 +1254,14 @@ func TestNewCommandWithErrors(t *testing.T) {
 			desc: "when the iam authn login query param contains multiple values",
 			args: []string{"proj:region:inst?auto-iam-authn=true&auto-iam-authn=false"},
 		},
+		{
+			desc: "when the sql-data query param contains multiple values",
+			args: []string{"proj:region:inst?sql-data=true&sql-data=false"},
+		},
+		{
+			desc: "when the sql-data query param is bogus",
+			args: []string{"proj:region:inst?sql-data=nope"},
+		},
 		{
 			desc: "when the iam authn login query param is bogus",
 			args: []string{"proj:region:inst?auto-iam-authn=nope"},
@@ -1185,6 +1295,13 @@ func TestNewCommandWithErrors(t *testing.T) {
 				"p:r:i",
 			},
 		},
+		{
+			desc: "using --private-ip with --sql-data",
+			args: []string{
+				"--private-ip", "--sql-data",
+				"p:r:i",
+			},
+		},
 		{
 			desc: "using private-ip query param with --auto-ip",
 			args: []string{

docs/cmd/cloud-sql-proxy.md

@@ -279,7 +279,9 @@ cloud-sql-proxy INSTANCE_CONNECTION_NAME... [flags]
                                              status code.
       --skip-failed-instance-config          If set, the Proxy will skip any instances that are invalid/unreachable (
                                              only applicable to Unix sockets)
+      --sql-data                             Enable SQL Data to tunnel through the Cloud SQL Admin API without needing network access to your public or private IP
       --sqladmin-api-endpoint string         API endpoint for all Cloud SQL Admin API requests. (default: https://sqladmin.googleapis.com)
+      --sqldata-api-endpoint string          Override the SQL Data API endpoint
   -l, --structured-logs                      Enable structured logging with LogEntry format
       --telemetry-prefix string              Prefix for Cloud Monitoring metrics.
       --telemetry-project string             Enable Cloud Monitoring and Cloud Trace with the provided project ID.