8000 feat(components): implement custom components operational health reporting by VLukiantsev · Pull Request #792 · google/xmc · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
8000
Diff view
2 changes: 2 additions & 0 deletions cmd/health_service/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ compat_go_library(
"//internal/storage/spanner/migrations/version",
"//internal/task_queue_redis/redisclient",
"//pkg/auth/oauth2",
"//pkg/components/store",
"//pkg/components/summary",
"@com_github_grpc_ecosystem_grpc_gateway_v2_custom//:runtime",
"@com_google_cloud_go_spanner//:spanner",
"@org_golang_google_api//cloudidentity/v1beta1:go_default_library",
Expand Down
3 changes: 3 additions & 0 deletions cmd/health_service/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ type Config struct {
CacheTTL time.Duration
ForceRefreshMaxRequests int
ForceRefreshWindow time.Duration
EnableComponents bool
}

var buildVersion string
Expand Down Expand Up @@ -73,6 +74,7 8000 @@ func init() {
configutil.DurationEnv(&cfg.CacheTTL, "health_status_cache_ttl", 15*time.Second)
configutil.IntEnv(&cfg.ForceRefreshMaxRequests, "health_status_force_refresh_max_requests", 5)
configutil.DurationEnv(&cfg.ForceRefreshWindow, "health_status_force_refresh_window", 5*time.Second)
configutil.BoolEnv(&cfg.EnableComponents, "enable_components", false)

defaultPort := os.Getenv("PORT")
if defaultPort == "" {
Expand All @@ -86,6 +88,7 @@ func init() {
flag.StringVar(&cfg.TLSCertFile, "tls_cert_file", "", "The TLS certificate file path.")
flag.StringVar(&cfg.TLSKeyFile, "tls_key_file", "", "The TLS private key file path.")
flag.StringVar(&cfg.ConfigDir, "config_dir", "/config", "The directory where ConfigMaps are mounted.")
flag.BoolVar(&cfg.EnableComponents, "enable_components", cfg.EnableComponents, "Enable custom components discovery in health service.")
}

// LoadConfig loads and returns the configuration.
Expand Down
75 changes: 73 additions & 2 deletions cmd/health_service/health_checker.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ import (

health "xmanager_cloud/common/health"
spb "xmanager_cloud/health_service/proto/api_go_grpc_proto"
"xmanager_cloud/logging/logging"
"xmanager_cloud/pkg/components/summary"

run "google.golang.org/api/run/v2"
"google.golang.org/grpc"
Expand All @@ -42,6 +44,7 @@ type HealthCheckerParams struct {
CloudRunService *run.Service
EssExpectedSchemaVersion int64
XidExpectedSchemaVersion int64
ComponentsProvider summary.CustomComponentsInfoProvider
}

type HealthChecker struct {
Expand All @@ -54,6 +57,7 @@ type HealthChecker struct {
cloudRunService *run.Service
essExpectedSchemaVersion int64
xidExpectedSchemaVersion int64
componentsProvider summary.CustomComponentsInfoProvider
}

func NewHealthChecker(params HealthCheckerParams) (*HealthChecker, error) {
Expand All @@ -80,6 +84,7 @@ func NewHealthChecker(params HealthCheckerParams) (*HealthChecker, error) {
cloudRunService: params.CloudRunService,
essExpectedSchemaVersion: params.EssExpectedSchemaVersion,
xidExpectedSchemaVersion: params.XidExpectedSchemaVersion,
componentsProvider: params.ComponentsProvider,
}, nil
}

Expand Down Expand Up @@ -112,10 +117,18 @@ func (c *HealthChecker) RunChecks(ctx context.Context) RunChecksResult {
}(i, target)
}

// 2. Run Spanner checks in parallel
// 2. Run Spanner and custom component checks in parallel
var essComp, xidComp *spb.ComponentHealth
var essVersion, xidVersion string
wg.Add(2)
var customCompHealths []summary.ComponentHealth
var customCompErr error

spannerWgCount := 2
if c.componentsProvider != nil {
spannerWgCount = 3
}
wg.Add(spannerWgCount)

go func() {
defer wg.Done()
essComp, essVersion = c.CheckSpannerESS(ctx)
Expand All @@ -124,19 +137,77 @@ func (c *HealthChecker) RunChecks(ctx context.Context) RunChecksResult {
defer wg.Done()
xidComp, xidVersion = c.CheckSpannerXID(ctx)
}()
if c.componentsProvider != nil {
go func() {
defer wg.Done()
compCtx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
customCompHealths, customCompErr = c.componentsProvider.GetComponentsHealth(compCtx)
}()
}
Comment thread
VLukiantsev marked this conversation as resolved.
Resolved

wg.Wait()

// 3. Append Spanner results to the components list
components = append(components, essComp, xidComp)

// 4. Append custom components health if provider is configured
if c.componentsProvider != nil {
if customCompErr != nil {
logging.Warn(ctx, "Failed to retrieve custom components health", "error", customCompErr)
}
components = appendCustomComponentHealth(components, customCompErr, customCompHealths)
}

return RunChecksResult{
Components: components,
EssDbSchemaVersion: essVersion,
XidDbSchemaVersion: xidVersion,
}
}

func appendCustomComponentHealth(components []*spb.ComponentHealth, err error, healths []summary.ComponentHealth) []*spb.ComponentHealth {
if err != nil {
return append(components, &spb.ComponentHealth{
Name: "Custom Components Provider",
Type: 8000 spb.ComponentType_COMPONENT_TYPE_CUSTOM_COMPONENT,
HealthState: spb.HealthState_HEALTH_STATE_UNHEALTHY,
ErrorMessage: fmt.Sprintf("failed to list custom components: %v", err),
})
}
for _, ch := range healths {
if ch.HealthState == summary.HealthStateDisabled {
continue
}
components = append(components, convertCustomComponentHealth(ch))
}
return components
}

func convertCustomComponentHealth(ch summary.ComponentHealth) *spb.ComponentHealth {
var hState spb.HealthState
var errMsg string

switch ch.HealthState {
case summary.HealthStateHealthy:
hState = spb.HealthState_HEALTH_STATE_HEALTHY
errMsg = ""
case summary.HealthStateDisabled:
Comment thread
andrzejfudro marked this conversation as resolved.
Resolved
hState = spb.HealthState_HEALTH_STATE_UNSPECIFIED
errMsg = ch.StatusMessage
default:
hState = spb.HealthState_HEALTH_STATE_UNHEALTHY
errMsg = ch.StatusMessage
}

return &spb.ComponentHealth{
Name: ch.Name,
Type: spb.ComponentType_COMPONENT_TYPE_CUSTOM_COMPONENT,
HealthState: hState,
ErrorMessage: errMsg,
}
}

type checkTarget struct {
name string
checkFn func(ctx context.Context) *spb.ComponentHealth
Expand Down
148 changes: 148 additions & 0 deletions cmd/health_service/health_checker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package main

import (
"context"
"errors"
"fmt"
"net"
"net/http"
Expand All @@ -25,6 +26,7 @@ import (

commonhealth "xmanager_cloud/common/health"
spb "xmanager_cloud/health_service/proto/api_go_grpc_proto"
"xmanager_cloud/pkg/components/summary"

"google.golang.org/api/option"
run "google.golang.org/api/run/v2"
Expand Down Expand Up @@ -665,3 +667,149 @@ func TestRunChecks(t *testing.T) {
})
}
}

type mockComponentsInfoProvider struct {
healthList []summary.ComponentHealth
healthErr error
}

func (m *mockComponentsInfoProvider) GetComponentsHealth(ctx context.Context) ([]summary.ComponentHealth, error) {
return m.healthList, m.healthErr
}

func TestHealthChecker_RunChecks_WithCustomComponents(t *testing.T) {
httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}))
defer httpServer.Close()

xidListener, err := net.Listen("tcp", "localhost:0")
if err != nil {
t.Fatalf("failed to create xid listener: %v", err)
}
defer xidListener.Close()

xidServer := grpc.NewServer()
xidHealthServer := health.NewServer()
xidHealthServer.SetServingStatus("xmc.XidService", grpc_health_v1.HealthCheckResponse_SERVING)
grpc_health_v1.RegisterHealthServer(xidServer, xidHealthServer)
go func() {
_ = xidServer.Serve(xidListener)
}()
defer xidServer.Stop()

mockProvider := &mockComponentsInfoProvider{
healthList: []summary.ComponentHealth{
{
Name: "clickhouse",
Type: summary.ComponentTypeCustom3PWrapper,
HealthState: summary.HealthStateHealthy,
StatusMessage: "Healthy and Operational",
},
{
Name: "gateway",
Type: summary.ComponentTypeCustom1PComponent,
HealthState: summary.HealthStateUnhealthy,
StatusMessage: "Pending GKE reconciliation",
},
{
Name: "redis-custom",
Type: summary.ComponentTypeCustom1PComponent,
HealthState: summary.HealthStateDisabled,
StatusMessage: "Component disabled",
},
},
}

checker, err := NewHealthChecker(HealthCheckerParams{
Config: &Config{
XidAddr: xidListener.Addr().String(),
EssURL: httpServer.URL,
ActuatorURL: httpServer.URL,
ReconcilerURL: httpServer.URL,
},
ComponentsProvider: mockProvider,
})
if err != nil {
t.Fatalf("failed to create health checker: %v", err)
}
defer checker.Close()

res := checker.RunChecks(context.Background())

customComps := make(map[string]*spb.ComponentHealth)
for _, comp := range res.Components {
if comp.Type == spb.ComponentType_COMPONENT_TYPE_CUSTOM_COMPONENT {
customComps[comp.Name] = comp
}
}

if len(customComps) != 2 {
t.Fatalf("expected 2 custom components (disabled filtered out), got %d", len(customComps))
}

ch, ok := customComps["clickhouse"]
if !ok || ch.HealthState != spb.HealthState_HEALTH_STATE_HEALTHY || ch.ErrorMessage != "" {
t.Errorf("unexpected clickhouse health: %+v", ch)
}

gw, ok := customComps["gateway"]
if !ok || gw.HealthState != spb.HealthState_HEALTH_STATE_UNHEALTHY || gw.ErrorMessage != "Pending GKE reconciliation" {
t.Errorf("unexpected gateway health: %+v", gw)
}
}

func TestHealthChecker_RunChecks_CustomComponentsError(t *testing.T) {
httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}))
defer httpServer.Close()

xidListener, err := net.Listen("tcp", "localhost:0")
if err != nil {
t.Fatalf("failed to create xid listener: %v", err)
}
defer xidListener.Close()

xidServer := grpc.NewServer()
xidHealthServer := health.NewServer()
xidHealthServer.SetServingStatus("xmc.XidService", grpc_health_v1.HealthCheckResponse_SERVING)
grpc_health_v1.RegisterHealthServer(xidServer, xidHealthServer)
go func() {
_ = xidServer.Serve(xidListener)
}()
defer xidServer.Stop()

mockProvider := &mockComponentsInfoProvider{
healthErr: errors.New("spanner connection failure"),
}

checker, err := NewHealthChecker(HealthCheckerParams{
Config: &Config{
XidAddr: xidListener.Addr().String(),
EssURL: httpServer.URL,
ActuatorURL: httpServer.URL,
ReconcilerURL: httpServer.URL,
},
ComponentsProvider: mockProvider,
})
if err != nil {
t.Fatalf("failed to create health checker: %v", err)
}
defer checker.Close()

// Should not crash, and should report Custom Components Provider as UNHEALTHY
res := ch 6C3F ecker.RunChecks(context.Background())
var customComps []*spb.ComponentHealth
for _, comp := range res.Components {
if comp != nil && comp.Type == spb.ComponentType_COMPONENT_TYPE_CUSTOM_COMPONENT {
customComps = append(customComps, comp)
}
}
if len(customComps) != 1 {
t.Fatalf("expected 1 custom component error placeholder, got %d", len(customComps))
}
if customComps[0].Name != "Custom Components Provider" || customComps[0].HealthState != spb.HealthState_HEALTH_STATE_UNHEALTHY {
t.Errorf("unexpected component health: %+v", customComps[0])
}
}
Loading
Loading
0