Skip to content

Commit 52fa30c

Browse files
iuwqyiriuwqyir
committed
Chain based table name and default columns configuration
1 parent 08bc042 commit 52fa30c

File tree

7 files changed

+239
-124
lines changed

7 files changed

+239
-124
lines changed

‎configs/config.go

Lines changed: 61 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
package config
22

33
import (
4+
"encoding/json"
45
"fmt"
6+
"os"
57
"strings"
68

79
"github.com/rs/zerolog/log"
@@ -63,15 +65,23 @@ type StorageConnectionConfig struct {
6365
Redis *RedisConfig `mapstructure:"redis"`
6466
}
6567

68+
type TableConfig struct {
69+
DefaultSelectFields []string `mapstructure:"defaultSelectFields"`
70+
TableName string `mapstructure:"tableName"`
71+
}
72+
73+
type TableOverrideConfig map[string]TableConfig
74+
6675
type ClickhouseConfig struct {
67-
Host string `mapstructure:"host"`
68-
Port int `mapstructure:"port"`
69-
Username string `mapstructure:"username"`
70-
Password string `mapstructure:"password"`
71-
Database string `mapstructure:"database"`
72-
DisableTLS bool `mapstructure:"disableTLS"`
73-
AsyncInsert bool `mapstructure:"asyncInsert"`
74-
MaxRowsPerInsert int `mapstructure:"maxRowsPerInsert"`
76+
Host string `mapstructure:"host"`
77+
Port int `mapstructure:"port"`
78+
Username string `mapstructure:"username"`
79+
Password string `mapstructure:"password"`
80+
Database string `mapstructure:"database"`
81+
DisableTLS bool `mapstructure:"disableTLS"`
82+
AsyncInsert bool `mapstructure:"asyncInsert"`
83+
MaxRowsPerInsert int `mapstructure:"maxRowsPerInsert"`
84+
ChainBasedConfig map[string]TableOverrideConfig `mapstructure:"chainBasedConfig"`
7585
}
7686

7787
type MemoryConfig struct {
@@ -160,5 +170,48 @@ func LoadConfig(cfgFile string) error {
160170
return fmt.Errorf("error unmarshalling config: %v", err)
161171
}
162172

173+
err = setCustomJSONConfigs()
174+
if err != nil {
175+
return fmt.Errorf("error setting custom JSON configs: %v", err)
176+
}
177+
178+
// Add debug logging
179+
if clickhouse := Cfg.Storage.Main.Clickhouse; clickhouse != nil {
180+
log.Debug().
181+
Interface("chainConfig", clickhouse.ChainBasedConfig).
182+
Msgf("Loaded chain config %v", clickhouse.ChainBasedConfig)
183+
}
184+
185+
return nil
186+
}
187+
188+
func setCustomJSONConfigs() error {
189+
if chainConfigJSON := os.Getenv("STORAGE_MAIN_CLICKHOUSE_CHAINBASEDCONFIG"); chainConfigJSON != "" {
190+
var mainChainConfig map[string]TableOverrideConfig
191+
if err := json.Unmarshal([]byte(chainConfigJSON), &mainChainConfig); err != nil {
192+
return fmt.Errorf("error parsing main chainBasedConfig JSON: %v", err)
193+
}
194+
if Cfg.Storage.Main.Clickhouse != nil {
195+
Cfg.Storage.Main.Clickhouse.ChainBasedConfig = mainChainConfig
196+
}
197+
}
198+
if chainConfigJSON := os.Getenv("STORAGE_STAGING_CLICKHOUSE_CHAINBASEDCONFIG"); chainConfigJSON != "" {
199+
var stagingChainConfig map[string]TableOverrideConfig
200+
if err := json.Unmarshal([]byte(chainConfigJSON), &stagingChainConfig); err != nil {
201+
return fmt.Errorf("error parsing staging chainBasedConfig JSON: %v", err)
202+
}
203+
if Cfg.Storage.Staging.Clickhouse != nil {
204+
Cfg.Storage.Staging.Clickhouse.ChainBasedConfig = stagingChainConfig
205+
}
206+
}
207+
if chainConfigJSON := os.Getenv("STORAGE_ORCHESTRATOR_CLICKHOUSE_CHAINBASEDCONFIG"); chainConfigJSON != "" {
208+
var orchestratorChainConfig map[string]TableOverrideConfig
209+
if err := json.Unmarshal([]byte(chainConfigJSON), &orchestratorChainConfig); err != nil {
210+
return fmt.Errorf("error parsing orchestrator chainBasedConfig JSON: %v", err)
211+
}
212+
if Cfg.Storage.Main.Clickhouse != nil {
213+
Cfg.Storage.Main.Clickhouse.ChainBasedConfig = orchestratorChainConfig
214+
}
215+
}
163216
return nil
164217
}

‎internal/common/BlockTimestamp.go

Lines changed: 0 additions & 30 deletions
This file was deleted.

‎internal/common/block.go

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -6,30 +6,30 @@ import (
66
)
77

88
type Block struct {
9-
ChainId *big.Int `json:"chain_id" ch:"chain_id"`
10-
Number *big.Int `json:"block_number" ch:"block_number"`
11-
Hash string `json:"hash" ch:"hash"`
12-
ParentHash string `json:"parent_hash" ch:"parent_hash"`
13-
Timestamp BlockTimestamp `json:"block_timestamp" ch:"block_timestamp"`
14-
Nonce string `json:"nonce" ch:"nonce"`
15-
Sha3Uncles string `json:"sha3_uncles" ch:"sha3_uncles"`
16-
MixHash string `json:"mix_hash" ch:"mix_hash"`
17-
Miner string `json:"miner" ch:"miner"`
18-
StateRoot string `json:"state_root" ch:"state_root"`
19-
TransactionsRoot string `json:"transactions_root" ch:"transactions_root"`
20-
ReceiptsRoot string `json:"receipts_root" ch:"receipts_root"`
21-
LogsBloom string `json:"logs_bloom" ch:"logs_bloom"`
22-
Size uint64 `json:"size" ch:"size"`
23-
ExtraData string `json:"extra_data" ch:"extra_data"`
24-
Difficulty *big.Int `json:"difficulty" ch:"difficulty"`
25-
TotalDifficulty *big.Int `json:"total_difficulty" ch:"total_difficulty"`
26-
TransactionCount uint64 `json:"transaction_count" ch:"transaction_count"`
27-
GasLimit *big.Int `json:"gas_limit" ch:"gas_limit"`
28-
GasUsed *big.Int `json:"gas_used" ch:"gas_used"`
29-
WithdrawalsRoot string `json:"withdrawals_root" ch:"withdrawals_root"`
30-
BaseFeePerGas uint64 `json:"base_fee_per_gas" ch:"base_fee_per_gas"`
31-
Sign int8 `json:"sign" ch:"sign"`
32-
InsertTimestamp time.Time `json:"insert_timestamp" ch:"insert_timestamp"`
9+
ChainId *big.Int `json:"chain_id" ch:"chain_id"`
10+
Number *big.Int `json:"block_number" ch:"block_number"`
11+
Hash string `json:"hash" ch:"hash"`
12+
ParentHash string `json:"parent_hash" ch:"parent_hash"`
13+
Timestamp time.Time `json:"block_timestamp" ch:"block_timestamp"`
14+
Nonce string `json:"nonce" ch:"nonce"`
15+
Sha3Uncles string `json:"sha3_uncles" ch:"sha3_uncles"`
16+
MixHash string `json:"mix_hash" ch:"mix_hash"`
17+
Miner string `json:"miner" ch:"miner"`
18+
StateRoot string `json:"state_root" ch:"state_root"`
19+
TransactionsRoot string `json:"transactions_root" ch:"transactions_root"`
20+
ReceiptsRoot string `json:"receipts_root" ch:"receipts_root"`
21+
LogsBloom string `json:"logs_bloom" ch:"logs_bloom"`
22+
Size uint64 `json:"size" ch:"size"`
23+
ExtraData string `json:"extra_data" ch:"extra_data"`
24+
Difficulty *big.Int `json:"difficulty" ch:"difficulty"`
25+
TotalDifficulty *big.Int `json:"total_difficulty" ch:"total_difficulty"`
26+
TransactionCount uint64 `json:"transaction_count" ch:"transaction_count"`
27+
GasLimit *big.Int `json:"gas_limit" ch:"gas_limit"`
28+
GasUsed *big.Int `json:"gas_used" ch:"gas_used"`
29+
WithdrawalsRoot string `json:"withdrawals_root" ch:"withdrawals_root"`
30+
BaseFeePerGas uint64 `json:"base_fee_per_gas" ch:"base_fee_per_gas"`
31+
Sign int8 `json:"sign" ch:"sign"`
32+
InsertTimestamp time.Time `json:"insert_timestamp" ch:"insert_timestamp"`
3333
}
3434

3535
type BlockData struct {

‎internal/common/log.go

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -13,21 +13,21 @@ import (
1313
)
1414

1515
type Log struct {
16-
ChainId *big.Int `json:"chain_id" ch:"chain_id" swaggertype:"string"`
17-
BlockNumber *big.Int `json:"block_number" ch:"block_number" swaggertype:"string"`
18-
BlockHash string `json:"block_hash" ch:"block_hash"`
19-
BlockTimestamp BlockTimestamp `json:"block_timestamp" ch:"block_timestamp"`
20-
TransactionHash string `json:"transaction_hash" ch:"transaction_hash"`
21-
TransactionIndex uint64 `json:"transaction_index" ch:"transaction_index"`
22-
LogIndex uint64 `json:"log_index" ch:"log_index"`
23-
Address string `json:"address" ch:"address"`
24-
Data string `json:"data" ch:"data"`
25-
Topic0 string `json:"topic_0" ch:"topic_0"`
26-
Topic1 string `json:"topic_1" ch:"topic_1"`
27-
Topic2 string `json:"topic_2" ch:"topic_2"`
28-
Topic3 string `json:"topic_3" ch:"topic_3"`
29-
Sign int8 `json:"sign" ch:"sign"`
30-
InsertTimestamp time.Time `json:"insert_timestamp" ch:"insert_timestamp"`
16+
ChainId *big.Int `json:"chain_id" ch:"chain_id" swaggertype:"string"`
17+
BlockNumber *big.Int `json:"block_number" ch:"block_number" swaggertype:"string"`
18+
BlockHash string `json:"block_hash" ch:"block_hash"`
19+
BlockTimestamp time.Time `json:"block_timestamp" ch:"block_timestamp"`
20+
TransactionHash string `json:"transaction_hash" ch:"transaction_hash"`
21+
TransactionIndex uint64 `json:"transaction_index" ch:"transaction_index"`
22+
LogIndex uint64 `json:"log_index" ch:"log_index"`
23+
Address string `json:"address" ch:"address"`
24+
Data string `json:"data" ch:"data"`
25+
Topic0 string `json:"topic_0" ch:"topic_0"`
26+
Topic1 string `json:"topic_1" ch:"topic_1"`
27+
Topic2 string `json:"topic_2" ch:"topic_2"`
28+
Topic3 string `json:"topic_3" ch:"topic_3"`
29+
Sign int8 `json:"sign" ch:"sign"`
30+
InsertTimestamp time.Time `json:"insert_timestamp" ch:"insert_timestamp"`
3131
}
3232

3333
func (l *Log) GetTopic(index int) (string, error) {

‎internal/common/transaction.go

Lines changed: 31 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -12,37 +12,37 @@ import (
1212
)
1313

1414
type Transaction struct {
15-
ChainId *big.Int `json:"chain_id" ch:"chain_id" swaggertype:"string"`
16-
Hash string `json:"hash" ch:"hash"`
17-
Nonce uint64 `json:"nonce" ch:"nonce"`
18-
BlockHash string `json:"block_hash" ch:"block_hash"`
19-
BlockNumber *big.Int `json:"block_number" ch:"block_number" swaggertype:"string"`
20-
BlockTimestamp BlockTimestamp `json:"block_timestamp" ch:"block_timestamp"`
21-
TransactionIndex uint64 `json:"transaction_index" ch:"transaction_index"`
22-
FromAddress string `json:"from_address" ch:"from_address"`
23-
ToAddress string `json:"to_address" ch:"to_address"`
24-
Value *big.Int `json:"value" ch:"value" swaggertype:"string"`
25-
Gas uint64 `json:"gas" ch:"gas"`
26-
GasPrice *big.Int `json:"gas_price" ch:"gas_price" swaggertype:"string"`
27-
Data string `json:"data" ch:"data"`
28-
FunctionSelector string `json:"function_selector" ch:"function_selector"`
29-
MaxFeePerGas *big.Int `json:"max_fee_per_gas" ch:"max_fee_per_gas" swaggertype:"string"`
30-
MaxPriorityFeePerGas *big.Int `json:"max_priority_fee_per_gas" ch:"max_priority_fee_per_gas" swaggertype:"string"`
31-
TransactionType uint8 `json:"transaction_type" ch:"transaction_type"`
32-
R *big.Int `json:"r" ch:"r" swaggertype:"string"`
33-
S *big.Int `json:"s" ch:"s" swaggertype:"string"`
34-
V *big.Int `json:"v" ch:"v" swaggertype:"string"`
35-
AccessListJson *string `json:"access_list_json" ch:"access_list"`
36-
ContractAddress *string `json:"contract_address" ch:"contract_address"`
37-
GasUsed *uint64 `json:"gas_used" ch:"gas_used"`
38-
CumulativeGasUsed *uint64 `json:"cumulative_gas_used" ch:"cumulative_gas_used"`
39-
EffectiveGasPrice *big.Int `json:"effective_gas_price" ch:"effective_gas_price" swaggertype:"string"`
40-
BlobGasUsed *uint64 `json:"blob_gas_used" ch:"blob_gas_used"`
41-
BlobGasPrice *big.Int `json:"blob_gas_price" ch:"blob_gas_price" swaggertype:"string"`
42-
LogsBloom *string `json:"logs_bloom" ch:"logs_bloom"`
43-
Status *uint64 `json:"status" ch:"status"`
44-
Sign int8 `json:"sign" ch:"sign"`
45-
InsertTimestamp time.Time `json:"insert_timestamp" ch:"insert_timestamp"`
15+
ChainId *big.Int `json:"chain_id" ch:"chain_id" swaggertype:"string"`
16+
Hash string `json:"hash" ch:"hash"`
17+
Nonce uint64 `json:"nonce" ch:"nonce"`
18+
BlockHash string `json:"block_hash" ch:"block_hash"`
19+
BlockNumber *big.Int `json:"block_number" ch:"block_number" swaggertype:"string"`
20+
BlockTimestamp time.Time `json:"block_timestamp" ch:"block_timestamp"`
21+
TransactionIndex uint64 `json:"transaction_index" ch:"transaction_index"`
22+
FromAddress string `json:"from_address" ch:"from_address"`
23+
ToAddress string `json:"to_address" ch:"to_address"`
24+
Value *big.Int `json:"value" ch:"value" swaggertype:"string"`
25+
Gas uint64 `json:"gas" ch:"gas"`
26+
GasPrice *big.Int `json:"gas_price" ch:"gas_price" swaggertype:"string"`
27+
Data string `json:"data" ch:"data"`
28+
FunctionSelector string `json:"function_selector" ch:"function_selector"`
29+
MaxFeePerGas *big.Int `json:"max_fee_per_gas" ch:"max_fee_per_gas" swaggertype:"string"`
30+
MaxPriorityFeePerGas *big.Int `json:"max_priority_fee_per_gas" ch:"max_priority_fee_per_gas" swaggertype:"string"`
31+
TransactionType uint8 `json:"transaction_type" ch:"transaction_type"`
32+
R *big.Int `json:"r" ch:"r" swaggertype:"string"`
33+
S *big.Int `json:"s" ch:"s" swaggertype:"string"`
34+
V *big.Int `json:"v" ch:"v" swaggertype:"string"`
35+
AccessListJson *string `json:"access_list_json" ch:"access_list"`
36+
ContractAddress *string `json:"contract_address" ch:"contract_address"`
37+
GasUsed *uint64 `json:"gas_used" ch:"gas_used"`
38+
CumulativeGasUsed *uint64 `json:"cumulative_gas_used" ch:"cumulative_gas_used"`
39+
EffectiveGasPrice *big.Int `json:"effective_gas_price" ch:"effective_gas_price" swaggertype:"string"`
40+
BlobGasUsed *uint64 `json:"blob_gas_used" ch:"blob_gas_used"`
41+
BlobGasPrice *big.Int `json:"blob_gas_price" ch:"blob_gas_price" swaggertype:"string"`
42+
LogsBloom *string `json:"logs_bloom" ch:"logs_bloom"`
43+
Status *uint64 `json:"status" ch:"status"`
44+
Sign int8 `json:"sign" ch:"sign"`
45+
InsertTimestamp time.Time `json:"insert_timestamp" ch:"insert_timestamp"`
4646
}
4747

4848
type DecodedTransactionData struct {

‎internal/rpc/serializer.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ func serializeBlock(chainId *big.Int, block common.RawBlock) common.Block {
119119
Number: hexToBigInt(block["number"]),
120120
Hash: interfaceToString(block["hash"]),
121121
ParentHash: interfaceToString(block["parentHash"]),
122-
Timestamp: common.BlockTimestamp{hexToTime(block["timestamp"])},
122+
Timestamp: hexToTime(block["timestamp"]),
123123
Nonce: interfaceToString(block["nonce"]),
124124
Sha3Uncles: interfaceToString(block["sha3Uncles"]),
125125
MixHash: interfaceToString(block["mixHash"]),
@@ -140,7 +140,7 @@ func serializeBlock(chainId *big.Int, block common.RawBlock) common.Block {
140140
}
141141
}
142142

143-
func serializeTransactions(chainId *big.Int, transactions []interface{}, blockTimestamp common.BlockTimestamp, receipts *common.RawReceipts) []common.Transaction {
143+
func serializeTransactions(chainId *big.Int, transactions []interface{}, blockTimestamp time.Time, receipts *common.RawReceipts) []common.Transaction {
144144
if len(transactions) == 0 {
145145
return []common.Transaction{}
146146
}
@@ -165,7 +165,7 @@ func serializeTransactions(chainId *big.Int, transactions []interface{}, blockTi
165165
return serializedTransactions
166166
}
167167

168-
func serializeTransaction(chainId *big.Int, tx map[string]interface{}, blockTimestamp common.BlockTimestamp, receipt *common.RawReceipt) common.Transaction {
168+
func serializeTransaction(chainId *big.Int, tx map[string]interface{}, blockTimestamp time.Time, receipt *common.RawReceipt) common.Transaction {
169169
return common.Transaction{
170170
ChainId: chainId,
171171
Hash: interfaceToString(tx["hash"]),
@@ -356,7 +356,7 @@ func serializeTrace(chainId *big.Int, trace map[string]interface{}, block common
356356
ChainID: chainId,
357357
BlockNumber: block.Number,
358358
BlockHash: block.Hash,
359-
BlockTimestamp: block.Timestamp.Time,
359+
BlockTimestamp: block.Timestamp,
360360
TransactionHash: interfaceToString(trace["transactionHash"]),
361361
TransactionIndex: func() uint64 {
362362
if v, ok := trace["transactionPosition"]; ok && v != nil {

0 commit comments

Comments
 (0)