|
| 1 | +package common |
| 2 | + |
| 3 | +import ( |
| 4 | + "context" |
| 5 | + _ "embed" |
| 6 | + "fmt" |
| 7 | + "os" |
| 8 | + "strings" |
| 9 | + "testing" |
| 10 | + "time" |
| 11 | + |
| 12 | + "github.com/ClickHouse/clickhouse-go/v2" |
| 13 | + "github.com/ClickHouse/clickhouse-go/v2/lib/driver" |
| 14 | + "github.com/minio/minio-go/v7" |
| 15 | + "github.com/minio/minio-go/v7/pkg/credentials" |
| 16 | + "github.com/stretchr/testify/require" |
| 17 | + "github.com/testcontainers/testcontainers-go" |
| 18 | + miniotest "github.com/testcontainers/testcontainers-go/modules/minio" |
| 19 | + "github.com/testcontainers/testcontainers-go/network" |
| 20 | +) |
| 21 | + |
| 22 | +type DockerSetup struct { |
| 23 | + Network *testcontainers.DockerNetwork |
| 24 | + Minio *miniotest.MinioContainer |
| 25 | + ClickHouse testcontainers.Container |
| 26 | +} |
| 27 | + |
| 28 | +func (setup *DockerSetup) CreateClickhouseClient(ctx context.Context, t *testing.T) (driver.Conn, error) { |
| 29 | + clickhousPort, err := setup.ClickHouse.MappedPort(ctx, "9000") |
| 30 | + |
| 31 | + if err != nil { |
| 32 | + return nil, err |
| 33 | + } |
| 34 | + |
| 35 | + opts, err := clickhouse.ParseDSN(fmt.Sprintf("tcp://default:test@localhost:%d/default", clickhousPort.Int())) |
| 36 | + |
| 37 | + if err != nil { |
| 38 | + return nil, err |
| 39 | + } |
| 40 | + |
| 41 | + conn, err := clickhouse.Open(opts) |
| 42 | + |
| 43 | + if err != nil { |
| 44 | + return nil, err |
| 45 | + } |
| 46 | + |
| 47 | + pingCtx, cancel := context.WithTimeout(ctx, time.Second*10) |
| 48 | + defer cancel() |
| 49 | + |
| 50 | +pingLoop: |
| 51 | + for { |
| 52 | + select { |
| 53 | + case <-pingCtx.Done(): |
| 54 | + return nil, pingCtx.Err() |
| 55 | + case <-time.After(time.Second): |
| 56 | + t.Log("Trying to ping ClickHouse server...") |
| 57 | + err = conn.Ping(ctx) |
| 58 | + if err == nil { |
| 59 | + break pingLoop |
| 60 | + } |
| 61 | + return nil, err |
| 62 | + } |
| 63 | + } |
| 64 | + |
| 65 | + ctx = clickhouse.Context( |
| 66 | + ctx, |
| 67 | + clickhouse.WithSettings(clickhouse.Settings{ |
| 68 | + "send_logs_level": "debug", |
| 69 | + }), |
| 70 | + clickhouse.WithLogs(func(l *clickhouse.Log) { |
| 71 | + if strings.Contains(l.Text, "Executable generates stderr:") { |
| 72 | + t.Log(l.Text) |
| 73 | + } |
| 74 | + }), |
| 75 | + ) |
| 76 | + |
| 77 | + return conn, nil |
| 78 | +} |
| 79 | + |
| 80 | +func WithDockerSetup( |
| 81 | + ctx context.Context, |
| 82 | + t *testing.T, |
| 83 | + bundlePath string, |
| 84 | + f func(*testing.T, *DockerSetup), |
| 85 | +) { |
| 86 | + net, err := network.New(ctx) |
| 87 | + require.NoError(t, err) |
| 88 | + testcontainers.CleanupNetwork(t, net) |
| 89 | + |
| 90 | + // start minion container |
| 91 | + minioContainer, err := miniotest.Run( |
| 92 | + ctx, |
| 93 | + "minio/minio:RELEASE.2024-01-16T16-07-38Z", |
| 94 | + testcontainers.CustomizeRequest(testcontainers.GenericContainerRequest{ |
| 95 | + ContainerRequest: testcontainers.ContainerRequest{ |
| 96 | + Networks: []string{net.Name}, |
| 97 | + NetworkAliases: map[string][]string{ |
| 98 | + net.Name: {"minio"}, |
| 99 | + }, |
| 100 | + }, |
| 101 | + }), |
| 102 | + miniotest.WithUsername("minio"), |
| 103 | + miniotest.WithPassword("minio123"), |
| 104 | + ) |
| 105 | + require.NoError(t, err) |
| 106 | + defer testcontainers.TerminateContainer(minioContainer) |
| 107 | + |
| 108 | + // create test bucket |
| 109 | + url, err := minioContainer.ConnectionString(ctx) |
| 110 | + require.NoError(t, err) |
| 111 | + minioClient, err := minio.New(url, &minio.Options{ |
| 112 | + Creds: credentials.NewStaticV4(minioContainer.Username, minioContainer.Password, ""), |
| 113 | + Secure: false, |
| 114 | + }) |
| 115 | + require.NoError(t, err) |
| 116 | + err = minioClient.MakeBucket(ctx, "test", minio.MakeBucketOptions{}) |
| 117 | + require.NoError(t, err) |
| 118 | + |
| 119 | + // start clickhouse-server container and mount UDF bundle |
| 120 | + bundleReader, err := os.Open(bundlePath) |
| 121 | + require.NoError(t, err) |
| 122 | + defer bundleReader.Close() |
| 123 | + clickhouseContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ |
| 124 | + ContainerRequest: testcontainers.ContainerRequest{ |
| 125 | + Image: "clickhouse/clickhouse-server:25.3", |
| 126 | + Networks: []string{net.Name}, |
| 127 | + NetworkAliases: map[string][]string{ |
| 128 | + net.Name: {"clickhouse"}, |
| 129 | + }, |
| 130 | + Env: map[string]string{ |
| 131 | + "CLICKHOUSE_PASSWORD": "test", |
| 132 | + "OBJSTR__S3__REGION": "us-east-1", |
| 133 | + "OBJSTR__S3__ACCESS_KEY_ID": "minio", |
| 134 | + "OBJSTR__S3__SECRET_ACCESS_KEY": "minio123", |
| 135 | + "OBJSTR__S3__DISABLE_SSL": "true", |
| 136 | + "OBJSTR__S3__FORCE_PATH_STYLE": "true", |
| 137 | + "OBJSTR__S3__ENDPOINT": "http://minio:9000", |
| 138 | + }, |
| 139 | + Files: []testcontainers.ContainerFile{ |
| 140 | + { |
| 141 | + Reader: bundleReader, |
| 142 | + HostFilePath: bundlePath, |
| 143 | + ContainerFilePath: "/bundle.tar.gz", |
| 144 | + FileMode: 700, |
| 145 | + }, |
| 146 | + }, |
| 147 | + }, |
| 148 | + Started: true, |
| 149 | + }) |
| 150 | + require.NoError(t, err) |
| 151 | + defer testcontainers.TerminateContainer(clickhouseContainer) |
| 152 | + |
| 153 | + // install UDF bundle inside clickhouse-server container |
| 154 | + c, _, err := clickhouseContainer.Exec(ctx, []string{"tar", "-xvzf", "/bundle.tar.gz", "-C", "/"}) |
| 155 | + require.NoError(t, err) |
| 156 | + require.Zero(t, c) |
| 157 | + |
| 158 | + f(t, &DockerSetup{ |
| 159 | + Network: net, |
| 160 | + Minio: minioContainer, |
| 161 | + ClickHouse: clickhouseContainer, |
| 162 | + }) |
| 163 | +} |
0 commit comments