In this article I will show you how to code a Go server to serve both gRPC and HTTP endpoints from a single service. This is not trivial on Cloud Run so it warrants sample code.

Typically to do this, you’d use the cmux package. However, cmux is a connection-level multiplexer, and if it determines a connection is HTTP/2 (e.g. gRPC) it sticks to that which causes regular HTTP requests to fail (and vice versa) because of the wrongly negotiated protocol. Since there’s a per-request load balancer between your Cloud Run app and your users, this protocol confusion due to using cmux on Cloud Run gives a subtle error:

upstream connect error or disconnect/reset before headers. reset reason: protocol

To work around this, we need to write our own HTTP/2 server (using h2c protocol) and instruct Cloud Run to send us all requests (even HTTP/1 ones over HTTP/2).

Here’s the complete code that multiplexes a sample gRPC service and a HTTP handler over the same port and does graceful shutdown for the unary requests.

package main

import (
	"context"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"

	"golang.org/x/net/http2"
	"golang.org/x/net/http2/h2c"
	"google.golang.org/grpc"
	"google.golang.org/grpc/health/grpc_health_v1"
)

func main() {
	grpcServ := grpc.NewServer()
	httpMux := http.NewServeMux()
	httpMux.HandleFunc("/", home)

	mySvc := &MyGrpcService{}
	grpc_health_v1.RegisterHealthServer(grpcServ, mySvc)

	ctx := context.Background()
	ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
	defer stop()

	mixedHandler := newHTTPandGRPCMux(httpMux, grpcServ)
	http2Server := &http2.Server{}
	http1Server := &http.Server{Handler: h2c.NewHandler(mixedHandler, http2Server)}
	lis, err := net.Listen("tcp", ":8080")
	if err != nil {
		panic(err)
	}

	err = http1Server.Serve(lis)
	if errors.Is(err, http.ErrServerClosed) {
		fmt.Println("server closed")
	} else if err != nil {
		panic(err)
	}
}

func home(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "hello from http handler!\n")
}

type MyGrpcService struct {
	grpc_health_v1.UnimplementedHealthServer
}

func (m *MyGrpcService) Check(_ context.Context, _ *grpc_health_v1.HealthCheckRequest) (*grpc_health_v1.HealthCheckResponse, error) {
	return &grpc_health_v1.HealthCheckResponse{Status: grpc_health_v1.HealthCheckResponse_SERVING}, nil
}

func (m *MyGrpcService) Watch(_ *grpc_health_v1.HealthCheckRequest, _ grpc_health_v1.Health_WatchServer) error {
	panic("not implemented")
}

func newHTTPandGRPCMux(httpHand http.Handler, grpcHandler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.ProtoMajor == 2 && strings.HasPrefix(r.Header.Get("content-type"), "application/grpc") {
			grpcHandler.ServeHTTP(w, r)
			return
		}
		httpHand.ServeHTTP(w, r)
	})
}

To deploy and test this application locally:

gcloud beta run deploy grpc-mux \
    --allow-unauthenticated \
    --use-http2 \
    --image=$(KO_DOCKER_REPO=gcr.io/ahmetb-demo ko publish .)

You can visit the https:// URL to verify HTTP endpoint. To verify the gRPC endpoint, you can use the grpc-health-probe tool:

$ grpc-health-probe -tls -addr [CLOUD_RUN_ADDRESS].a.run.app:443
status: SERVING

Hopefully that’s useful to someone!