Skip to content

Commit 5fde0b1

Browse files
jsbroksjsbroks
authored
Merge pull request #20 from ctrlplanedev/kubernetes-scanning
2 parents a9d4ee7 + 414548d commit 5fde0b1

File tree

7 files changed

+340
-60
lines changed

7 files changed

+340
-60
lines changed

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
.env
2+
test.bash

‎cmd/ctrlc/root/sync/aws/ec2/ec2.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -121,8 +121,8 @@ func NewSyncEC2Cmd() *cobra.Command {
121121
}
122122

123123
instanceData := EC2Instance{
124-
ID: *instance.InstanceId,
125-
Name: name,
124+
ID: *instance.InstanceId,
125+
Name: name,
126126
ConnectionMethod: ConnectionMethod{
127127
Type: "aws",
128128
Region: region,

‎cmd/ctrlc/root/sync/clickhouse/clickhouse.go‎

Lines changed: 198 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
package clickhouse
22

33
import (
4+
"bytes"
45
"context"
56
"encoding/json"
67
"fmt"
8+
"io"
79
"net/http"
810
"os"
911
"strings"
@@ -16,23 +18,85 @@ import (
1618
"github.com/spf13/viper"
1719
)
1820

19-
type ClickHouseConfig struct {
20-
ID string `json:"id"`
21-
Name string `json:"name"`
22-
State string `json:"state"`
23-
Region string `json:"region"`
24-
CloudProvider string `json:"cloudProvider"`
25-
Tier string `json:"tier"`
26-
IdleScaling map[string]interface{} `json:"idleScaling"`
27-
TotalDiskSize int `json:"totalDiskSize"`
28-
TotalMemoryMB int `json:"totalMemoryMB"`
29-
MinTotalMemory int `json:"minTotalMemory"`
30-
MaxTotalMemory int `json:"maxTotalMemory"`
31-
Created string `json:"created"`
32-
Endpoints []map[string]interface{} `json:"endpoints"`
21+
type ClickhouseEndpointResponse struct {
22+
Protocol string `json:"protocol"`
23+
Host string `json:"host"`
24+
Port int `json:"port"`
25+
Username string `json:"username"`
3326
}
3427

35-
func (c *ClickHouseConfig) Struct() map[string]interface{} {
28+
type IPAccessListItem struct {
29+
Source string `json:"source"`
30+
Description string `json:"description"`
31+
}
32+
33+
type ClickHouseConfigResponse struct {
34+
ID string `json:"id"`
35+
Name string `json:"name"`
36+
Provider string `json:"provider"`
37+
Region string `json:"region"`
38+
State string `json:"state"`
39+
Endpoints []ClickhouseEndpointResponse `json:"endpoints"`
40+
Tier string `json:"tier"`
41+
MinTotalMemoryGb int `json:"minTotalMemoryGb"`
42+
MaxTotalMemoryGb int `json:"maxTotalMemoryGb"`
43+
MinReplicaMemoryGb int `json:"minReplicaMemoryGb"`
44+
MaxReplicaMemoryGb int `json:"maxReplicaMemoryGb"`
45+
NumReplicas int `json:"numReplicas"`
46+
IdleScaling bool `json:"idleScaling"`
47+
IdleTimeoutMinutes int `json:"idleTimeoutMinutes"`
48+
IPAccessList []IPAccessListItem `json:"ipAccessList"`
49+
CreatedAt string `json:"createdAt"`
50+
EncryptionKey string `json:"encryptionKey"`
51+
EncryptionAssumedRoleIdentifier string `json:"encryptionAssumedRoleIdentifier"`
52+
IamRole string `json:"iamRole"`
53+
PrivateEndpointIds []string `json:"privateEndpointIds"`
54+
AvailablePrivateEndpointIds []string `json:"availablePrivateEndpointIds"`
55+
DataWarehouseId string `json:"dataWarehouseId"`
56+
IsPrimary bool `json:"isPrimary"`
57+
IsReadonly bool `json:"isReadonly"`
58+
ReleaseChannel string `json:"releaseChannel"`
59+
ByocId string `json:"byocId"`
60+
HasTransparentDataEncryption bool `json:"hasTransparentDataEncryption"`
61+
TransparentDataEncryptionKeyId string `json:"transparentDataEncryptionKeyId"`
62+
EncryptionRoleId string `json:"encryptionRoleId"`
63+
}
64+
65+
type Connection struct {
66+
Host string
67+
Port int
68+
Username string
69+
}
70+
71+
func (c *ClickHouseConfigResponse) GetConnection() Connection {
72+
for _, endpoint := range c.Endpoints {
73+
if endpoint.Protocol == "https" {
74+
return Connection{
75+
Host: endpoint.Host,
76+
Port: endpoint.Port,
77+
Username: endpoint.Username,
78+
}
79+
}
80+
}
81+
for _, endpoint := range c.Endpoints {
82+
if endpoint.Protocol == "native" {
83+
return Connection{
84+
Host: endpoint.Host,
85+
Port: endpoint.Port,
86+
Username: endpoint.Username,
87+
}
88+
}
89+
}
90+
return Connection{}
91+
}
92+
93+
type ClickHouseListResponse struct {
94+
Status int `json:"status"`
95+
RequestId string `json:"requestId"`
96+
Result []ClickHouseConfigResponse `json:"result"`
97+
}
98+
99+
func (c *ClickHouseConfigResponse) Struct() map[string]interface{} {
36100
b, _ := json.Marshal(c)
37101
var m map[string]interface{}
38102
json.Unmarshal(b, &m)
@@ -42,14 +106,16 @@ func (c *ClickHouseConfig) Struct() map[string]interface{} {
42106
type ClickHouseClient struct {
43107
httpClient *http.Client
44108
apiUrl string
109+
apiId string
45110
apiKey string
46111
organizationID string
47112
}
48113

49-
func NewClickHouseClient(apiUrl, apiKey, organizationID string) *ClickHouseClient {
114+
func NewClickHouseClient(apiUrl, apiId, apiKey, organizationID string) *ClickHouseClient {
50115
return &ClickHouseClient{
51116
httpClient: &http.Client{},
52117
apiUrl: apiUrl,
118+
apiId: apiId,
53119
apiKey: apiKey,
54120
organizationID: organizationID,
55121
}
@@ -60,29 +126,30 @@ type ServiceList struct {
60126
}
61127

62128
type Service struct {
63-
ID string `json:"id"`
64-
Name string `json:"name"`
65-
State string `json:"state"`
66-
Region string `json:"region"`
67-
CloudProvider string `json:"cloudProvider"`
68-
Tier string `json:"tier"`
69-
IdleScaling map[string]interface{} `json:"idleScaling"`
70-
TotalDiskSize int `json:"totalDiskSize"`
71-
TotalMemoryMB int `json:"totalMemoryMB"`
72-
MinTotalMemory int `json:"minTotalMemory"`
73-
MaxTotalMemory int `json:"maxTotalMemory"`
74-
Created string `json:"created"`
75-
Endpoints []map[string]interface{} `json:"endpoints"`
129+
ID string `json:"id"`
130+
Name string `json:"name"`
131+
State string `json:"state"`
132+
Region string `json:"region"`
133+
CloudProvider string `json:"cloudProvider"`
134+
Tier string `json:"tier"`
135+
IdleScaling map[string]any `json:"idleScaling"`
136+
TotalDiskSize int `json:"totalDiskSize"`
137+
TotalMemoryMB int `json:"totalMemoryMB"`
138+
MinTotalMemory int `json:"minTotalMemory"`
139+
MaxTotalMemory int `json:"maxTotalMemory"`
140+
Created string `json:"created"`
141+
Endpoints []map[string]any `json:"endpoints"`
76142
}
77143

78-
func (c *ClickHouseClient) GetServices(ctx context.Context) ([]Service, error) {
144+
func (c *ClickHouseClient) GetServices(ctx context.Context) ([]ClickHouseConfigResponse, error) {
145+
log.Info("Getting services for organization", "organizationID", c.organizationID)
79146
url := fmt.Sprintf("%s/v1/organizations/%s/services", c.apiUrl, c.organizationID)
80147
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
81148
if err != nil {
82149
return nil, fmt.Errorf("failed to create request: %w", err)
83150
}
84151

85-
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", c.apiKey))
152+
req.SetBasicAuth(c.apiId, c.apiKey)
86153
req.Header.Set("Content-Type", "application/json")
87154

88155
resp, err := c.httpClient.Do(req)
@@ -92,36 +159,67 @@ func (c *ClickHouseClient) GetServices(ctx context.Context) ([]Service, error) {
92159
defer resp.Body.Close()
93160

94161
if resp.StatusCode != http.StatusOK {
162+
log.Error("Unexpected status code", "status", resp.StatusCode)
163+
body, _ := io.ReadAll(resp.Body)
164+
log.Error("Response body", "body", string(body))
95165
return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
96166
}
97167

98-
var result ServiceList
168+
// Print raw response for debugging
169+
body, err := io.ReadAll(resp.Body)
170+
if err != nil {
171+
return nil, fmt.Errorf("failed to read response body: %w", err)
172+
}
173+
174+
// Pretty print the JSON
175+
var prettyJSON bytes.Buffer
176+
if err := json.Indent(&prettyJSON, body, "", " "); err != nil {
177+
return nil, fmt.Errorf("failed to pretty print JSON: %w", err)
178+
}
179+
log.Info("Raw response:", "body", prettyJSON.String())
180+
181+
// Reset the response body for subsequent reading
182+
resp.Body = io.NopCloser(bytes.NewBuffer(body))
183+
var result ClickHouseListResponse
99184
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
100185
return nil, fmt.Errorf("failed to decode response: %w", err)
101186
}
102187

103-
return result.Services, nil
188+
return result.Result, nil
104189
}
105190

106191
func NewSyncClickhouseCmd() *cobra.Command {
107192
var providerName string
108193
var clickhouseApiUrl string
109-
var clickhouseApiKey string
194+
var clickhouseApiSecret string
195+
var clickhouseApiId string
110196
var organizationID string
111197

112198
cmd := &cobra.Command{
113199
Use: "clickhouse",
114200
Short: "Sync ClickHouse instances into Ctrlplane",
115201
Example: heredoc.Doc(`
116-
$ ctrlc sync clickhouse --workspace 2a7c5560-75c9-4dbe-be74-04ee33bf8188
202+
$ ctrlc sync clickhouse
117203
`),
118204
PreRunE: func(cmd *cobra.Command, args []string) error {
119-
if clickhouseApiKey == "" {
120-
return fmt.Errorf("clickhouse-key must be provided")
205+
if clickhouseApiSecret == "" {
206+
clickhouseApiSecret = os.Getenv("CLICKHOUSE_API_SECRET")
207+
}
208+
if clickhouseApiSecret == "" {
209+
return fmt.Errorf("clickhouse-secret must be provided")
210+
}
211+
if organizationID == "" {
212+
organizationID = os.Getenv("CLICKHOUSE_ORGANIZATION_ID")
121213
}
122214
if organizationID == "" {
123215
return fmt.Errorf("organization-id must be provided")
124216
}
217+
if clickhouseApiId == "" {
218+
clickhouseApiId = os.Getenv("CLICKHOUSE_API_ID")
219+
}
220+
if clickhouseApiId == "" {
221+
return fmt.Errorf("clickhouse-api-id must be provided")
222+
}
125223
return nil
126224
},
127225
RunE: func(cmd *cobra.Command, args []string) error {
@@ -133,34 +231,80 @@ func NewSyncClickhouseCmd() *cobra.Command {
133231
if err != nil {
134232
return fmt.Errorf("failed to create API client: %w", err)
135233
}
136-
chClient := NewClickHouseClient(clickhouseApiUrl, clickhouseApiKey, organizationID)
234+
235+
chClient := NewClickHouseClient(clickhouseApiUrl, clickhouseApiId, clickhouseApiSecret, organizationID)
137236
ctx := context.Background()
138237
services, err := chClient.GetServices(ctx)
139238
if err != nil {
140239
return fmt.Errorf("failed to list ClickHouse services: %w", err)
141240
}
241+
142242
resources := []api.AgentResource{}
143243
for _, service := range services {
144-
metadata := map[string]string{}
145-
metadata["clickhouse/id"] = service.ID
146-
metadata["clickhouse/name"] = service.Name
147-
metadata["clickhouse/state"] = service.State
148-
metadata["clickhouse/region"] = service.Region
149-
metadata["clickhouse/cloud-provider"] = service.CloudProvider
150-
metadata["clickhouse/tier"] = service.Tier
151-
metadata["clickhouse/created"] = service.Created
244+
var endpoints []string
245+
for _, endpoint := range service.Endpoints {
246+
endpointString := fmt.Sprintf("%s://%s:%d", endpoint.Protocol, endpoint.Host, endpoint.Port)
247+
endpoints = append(endpoints, endpointString)
248+
}
249+
connection := service.GetConnection()
250+
metadata := map[string]string{
251+
"database/id": service.ID,
252+
"database/model": "relational",
253+
"database/port": fmt.Sprintf("%d", connection.Port),
254+
"database/host": connection.Host,
152255

153-
config := ClickHouseConfig(service) // Direct type conversion since fields match
256+
"clickhouse/id": service.ID,
257+
"clickhouse/name": service.Name,
258+
"clickhouse/state": service.State,
259+
"clickhouse/region": service.Region,
260+
"clickhouse/tier": service.Tier,
261+
"clickhouse/endpoints": strings.Join(endpoints, ","),
262+
"clickhouse/data-warehouse-id": service.DataWarehouseId,
263+
"clickhouse/is-primary": fmt.Sprintf("%t", service.IsPrimary),
264+
"clickhouse/is-readonly": fmt.Sprintf("%t", service.IsReadonly),
265+
"clickhouse/release-channel": service.ReleaseChannel,
266+
"clickhouse/encryption-key": service.EncryptionKey,
267+
"clickhouse/encryption-assumed-role-identifier": service.EncryptionAssumedRoleIdentifier,
268+
"clickhouse/encryption-role-id": service.EncryptionRoleId,
269+
"clickhouse/has-transparent-data-encryption": fmt.Sprintf("%t", service.HasTransparentDataEncryption),
270+
"clickhouse/transparent-data-encryption-key-id": service.TransparentDataEncryptionKeyId,
271+
"clickhouse/iam-role": service.IamRole,
272+
"clickhouse/byoc-id": service.ByocId,
273+
274+
"clickhouse/min-total-memory-gb": fmt.Sprintf("%d", service.MinTotalMemoryGb),
275+
"clickhouse/max-total-memory-gb": fmt.Sprintf("%d", service.MaxTotalMemoryGb),
276+
"clickhouse/min-replica-memory-gb": fmt.Sprintf("%d", service.MinReplicaMemoryGb),
277+
"clickhouse/max-replica-memory-gb": fmt.Sprintf("%d", service.MaxReplicaMemoryGb),
278+
"clickhouse/num-replicas": fmt.Sprintf("%d", service.NumReplicas),
279+
"clickhouse/idle-scaling": fmt.Sprintf("%t", service.IdleScaling),
280+
"clickhouse/idle-timeout-minutes": fmt.Sprintf("%d", service.IdleTimeoutMinutes),
281+
}
154282

155283
// Create a sanitized name
156284
name := strings.Split(service.Name, ".")[0]
157285
resources = append(resources, api.AgentResource{
158-
Version: "clickhouse/v1",
159-
Kind: "Service",
286+
Version: "https://schema.ctrlplane.dev/database/v1",
287+
Kind: "ClickhouseCloud",
160288
Name: name,
161289
Identifier: fmt.Sprintf("%s/%s", organizationID, service.ID),
162-
Config: config.Struct(),
163-
Metadata: metadata,
290+
Config: map[string]any{
291+
"host": connection.Host,
292+
"port": connection.Port,
293+
"username": connection.Username,
294+
295+
"clickhouse": map[string]any{
296+
"id": service.ID,
297+
"name": service.Name,
298+
"state": service.State,
299+
"provider": service.Provider,
300+
"region": service.Region,
301+
"endpoints": service.Endpoints,
302+
"iamRole": service.IamRole,
303+
"isPrimary": service.IsPrimary,
304+
"isReadonly": service.IsReadonly,
305+
},
306+
},
307+
Metadata: metadata,
164308
})
165309
}
166310
log.Info("Upserting resources", "count", len(resources))
@@ -180,10 +324,9 @@ func NewSyncClickhouseCmd() *cobra.Command {
180324

181325
cmd.Flags().StringVarP(&providerName, "provider", "p", "clickhouse", "The name of the provider to use")
182326
cmd.Flags().StringVarP(&clickhouseApiUrl, "clickhouse-url", "u", "https://api.clickhouse.cloud", "The URL of the ClickHouse API")
183-
cmd.Flags().StringVarP(&clickhouseApiKey, "clickhouse-key", "k", os.Getenv("CLICKHOUSE_API_KEY"), "The API key to use")
184-
cmd.Flags().StringVarP(&organizationID, "organization-id", "o", os.Getenv("CLICKHOUSE_ORGANIZATION_ID"), "The ClickHouse organization ID")
185-
186-
cmd.MarkFlagRequired("organization-id")
327+
cmd.Flags().StringVarP(&clickhouseApiSecret, "clickhouse-secret", "s", "", "The API secret to use")
328+
cmd.Flags().StringVarP(&clickhouseApiId, "clickhouse-api-id", "", "", "The API ID to use")
329+
cmd.Flags().StringVarP(&organizationID, "organization-id", "o", "", "The ClickHouse organization ID")
187330

188331
return cmd
189332
}

0 commit comments

Comments
 (0)