Skip to content

Commit b670fa1

Browse files
authored
Merge pull request #2 from kintsugi-tech/luca/fix-migration
Luca/fix migration
2 parents ee63714 + c3b2976 commit b670fa1

10 files changed

Lines changed: 270 additions & 26 deletions

File tree

cmd/migrate/cmd.go

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,15 @@ import (
88
"github.com/spf13/cobra"
99

1010
v3 "github.com/forbole/callisto/v4/cmd/migrate/v3"
11-
v6 "github.com/forbole/callisto/v4/cmd/migrate/v5"
11+
v5 "github.com/forbole/callisto/v4/cmd/migrate/v5"
1212
)
1313

1414
type Migrator func(parseCfg *parsecmdtypes.Config) error
1515

1616
var (
1717
migrations = map[string]Migrator{
1818
"v3": v3.RunMigration,
19-
"v6": v6.RunMigration,
19+
"v5": v5.RunMigration,
2020
}
2121
)
2222

@@ -34,8 +34,7 @@ func NewMigrateCmd(appName string, parseConfig *parsecmdtypes.Config) *cobra.Com
3434
Use: "migrate [to-version]",
3535
Short: "Perform the migrations from the current version to the specified one",
3636
Long: `Migrates all the necessary things (config file, database, etc) from the current version to the new one.
37-
If you are upgrading from a very old version to the latest one, migrations must be performed in order
38-
(eg. to migrate from v1 to v3 you need to do v1 -> v2 and then v2 -> v3).
37+
Note that migrations must be performed in order: to migrate from vX to vX+2 you need to do vX -> vX+1 and then vX+1 -> vX+2.
3938
`,
4039
Example: fmt.Sprintf("%s migrate v3", appName),
4140
Args: cobra.RangeArgs(0, 1),

database/database_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,9 @@ type DbTestSuite struct {
4242
}
4343

4444
func (suite *DbTestSuite) SetupTest() {
45-
4645
// Build the database
4746
dbCfg := dbconfig.NewDatabaseConfig(
48-
"postgresql://bdjuno:password@localhost:6433/bdjuno?sslmode=disable&search_path=public", //TODO: check if ok
47+
"postgresql://callisto:password@localhost:6433/callisto?sslmode=disable&search_path=public", //TODO: check if ok
4948
"false",
5049
"",
5150
"",
@@ -58,6 +57,7 @@ func (suite *DbTestSuite) SetupTest() {
5857

5958
db, err := database.Builder(utils.GetCodec())(junodb.NewContext(dbCfg, logging.DefaultLogger()))
6059
suite.Require().NoError(err)
60+
6161
bigDipperDb, ok := (db).(*database.Db)
6262
suite.Require().True(ok)
6363

database/gov.go

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ func (db *Db) SaveProposals(proposals []types.Proposal) error {
101101

102102
proposalsQuery := `
103103
INSERT INTO proposal(
104-
id, title, description, metadata, content, proposer_address, status,
104+
id, title, description, metadata, content, proposal_type, proposer_address, status,
105105
submit_time, deposit_end_time, voting_start_time, voting_end_time
106106
) VALUES`
107107
var proposalsParams []interface{}
@@ -112,16 +112,29 @@ INSERT INTO proposal(
112112

113113
// Prepare the proposal query
114114
vi := i * 11
115-
proposalsQuery += fmt.Sprintf("($%d,$%d,$%d,$%d,$%d,$%d,$%d,$%d,$%d,$%d,$%d),",
116-
vi+1, vi+2, vi+3, vi+4, vi+5, vi+6, vi+7, vi+8, vi+9, vi+10, vi+11)
117-
115+
proposalsQuery += fmt.Sprintf("($%d,$%d,$%d,$%d,$%d,$%d,$%d,$%d,$%d,$%d,$%d,$%d),",
116+
vi+1, vi+2, vi+3, vi+4, vi+5, vi+6, vi+7, vi+8, vi+9, vi+10, vi+11, vi+12)
117+
var proposalType string
118118
var jsonMessages []string
119+
119120
for _, msg := range proposal.Messages {
120121
contentBz, err := db.cdc.MarshalJSON(msg)
121122
if err != nil {
122123
return fmt.Errorf("error while marshalling proposal msg: %s", err)
123124
}
124125
jsonMessages = append(jsonMessages, string(contentBz))
126+
127+
if proposalType == "" {
128+
var msgMap map[string]interface{}
129+
if err := json.Unmarshal(contentBz, &msgMap); err != nil {
130+
return fmt.Errorf("error unmarshalling proposal msg for type extraction: %s", err)
131+
}
132+
if t, ok := msgMap["@type"].(string); ok {
133+
proposalType = t
134+
} else {
135+
proposalType = ""
136+
}
137+
}
125138
}
126139

127140
proposalsParams = append(proposalsParams,
@@ -130,6 +143,7 @@ INSERT INTO proposal(
130143
proposal.Summary,
131144
proposal.Metadata,
132145
fmt.Sprintf("[%s]", strings.Join(jsonMessages, ",")),
146+
proposalType,
133147
proposal.Proposer,
134148
proposal.Status,
135149
proposal.SubmitTime,

database/gov_test.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,7 @@ func (suite *DbTestSuite) TestBigDipperDb_SaveProposals() {
187187
"Proposal Description 1",
188188
"Proposal Metadata 1",
189189
"[{\"@type\": \"/cosmos.gov.v1.MsgUpdateParams\", \"params\": {\"quorum\": \"0.5\", \"threshold\": \"0.3\", \"min_deposit\": [{\"denom\": \"uatom\", \"amount\": \"1000\"}], \"voting_period\": \"0.000300s\", \"burn_vote_veto\": false, \"veto_threshold\": \"0.15\", \"burn_vote_quorum\": false, \"min_deposit_ratio\": \"\", \"max_deposit_period\": \"300s\", \"expedited_threshold\": \"\", \"proposal_cancel_dest\": \"\", \"expedited_min_deposit\": [], \"proposal_cancel_ratio\": \"\", \"expedited_voting_period\": null, \"min_initial_deposit_ratio\": \"0\", \"burn_proposal_deposit_prevote\": false}, \"authority\": \"cosmos10d07y265gmmuvt4z0w9aw880jnsr700j6zn9kn\"}]",
190+
"cosmos.gov.v1.MsgUpdateParams",
190191
time.Date(2020, 1, 1, 00, 00, 00, 000, time.UTC),
191192
time.Date(2020, 1, 1, 01, 00, 00, 000, time.UTC),
192193
testutils.NewTimePointer(time.Date(2020, 1, 1, 02, 00, 00, 000, time.UTC)),
@@ -200,6 +201,7 @@ func (suite *DbTestSuite) TestBigDipperDb_SaveProposals() {
200201
"Proposal Description 2",
201202
"Proposal Metadata 2",
202203
"[]",
204+
"cosmos.gov.v1.MsgUpdateParams",
203205
time.Date(2020, 1, 2, 00, 00, 00, 000, time.UTC),
204206
time.Date(2020, 1, 2, 01, 00, 00, 000, time.UTC),
205207
testutils.NewTimePointer(time.Date(2020, 1, 2, 02, 00, 00, 000, time.UTC)),
@@ -362,6 +364,7 @@ func (suite *DbTestSuite) TestBigDipperDb_UpdateProposal() {
362364
"Description of proposal 1",
363365
"Metadata of proposal 1",
364366
"[{\"@type\": \"/cosmos.gov.v1.MsgUpdateParams\", \"params\": {\"quorum\": \"0.5\", \"threshold\": \"0.3\", \"min_deposit\": [{\"denom\": \"uatom\", \"amount\": \"1000\"}], \"voting_period\": \"0.000300s\", \"burn_vote_veto\": false, \"veto_threshold\": \"0.15\", \"burn_vote_quorum\": false, \"min_deposit_ratio\": \"\", \"max_deposit_period\": \"300s\", \"expedited_threshold\": \"\", \"proposal_cancel_dest\": \"\", \"expedited_min_deposit\": [], \"proposal_cancel_ratio\": \"\", \"expedited_voting_period\": null, \"min_initial_deposit_ratio\": \"0\", \"burn_proposal_deposit_prevote\": false}, \"authority\": \"cosmos10d07y265gmmuvt4z0w9aw880jnsr700j6zn9kn\"}]",
367+
"cosmos.gov.v1.MsgUpdateParams",
365368
proposal.SubmitTime,
366369
proposal.DepositEndTime,
367370
timestamp1,

database/migrate/v5/migrate.go

Lines changed: 42 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ package v5
22

33
import (
44
"fmt"
5+
"log"
6+
"os"
57

68
utils "github.com/forbole/callisto/v4/modules/utils"
79
"github.com/forbole/callisto/v4/types"
@@ -26,6 +28,17 @@ func (db *Migrator) Migrate() error {
2628
return err
2729
}
2830
}
31+
// read the sql migration file
32+
sqlBytes, err := os.ReadFile("database/migrate/v5/migrationv5.sql")
33+
if err != nil {
34+
log.Fatalf("Failed to read SQL file: %v", err)
35+
}
36+
37+
// execute scripts
38+
err = db.migrateDbSchema(string(sqlBytes))
39+
if err != nil {
40+
log.Fatalf("Migration failed: %v", err)
41+
}
2942
return nil
3043
}
3144

@@ -37,17 +50,40 @@ func (db *Migrator) getMsgTypesFromMessageTable() ([]MessageRow, error) {
3750
if err != nil {
3851
return nil, err
3952
}
40-
4153
return rows, nil
4254
}
4355

4456
// migrateMsgTypes stores the given message type inside the database
4557
func (db *Migrator) migrateMsgTypes(msg *types.MessageType) error {
46-
stmt := `
47-
INSERT INTO message_type(type, module, label, height)
48-
VALUES ($1, $2, $3, $4)
49-
ON CONFLICT (type) DO NOTHING`
58+
// Step 1: Create the table and indexes (no parameters, can be run as plain exec)
59+
schemaStmt := `
60+
CREATE TABLE IF NOT EXISTS message_type
61+
(
62+
type TEXT NOT NULL UNIQUE,
63+
module TEXT NOT NULL,
64+
label TEXT NOT NULL,
65+
height BIGINT NOT NULL
66+
);
67+
CREATE INDEX IF NOT EXISTS message_type_module_index ON message_type (module);
68+
CREATE INDEX IF NOT EXISTS message_type_type_index ON message_type (type);
69+
`
70+
71+
_, err := db.SQL.Exec(schemaStmt)
72+
if err != nil {
73+
return err
74+
}
75+
76+
// Step 2: Insert the message type (with parameters)
77+
insertStmt := `
78+
INSERT INTO message_type(type, module, label, height)
79+
VALUES ($1, $2, $3, $4)
80+
ON CONFLICT (type) DO NOTHING;`
81+
82+
_, err = db.SQL.Exec(insertStmt, msg.Type, msg.Module, msg.Label, msg.Height)
83+
return err
84+
}
5085

51-
_, err := db.SQL.Exec(stmt, msg.Type, msg.Module, msg.Label, msg.Height)
86+
func (db *Migrator) migrateDbSchema(migrateSQL string) error {
87+
_, err := db.SQL.Exec(migrateSQL)
5288
return err
5389
}
Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
/*######################### cosmos00.sql ##########################*/
2+
3+
/* change vacuum rules */
4+
ALTER TABLE block
5+
SET (
6+
autovacuum_vacuum_scale_factor = 0,
7+
autovacuum_analyze_scale_factor = 0,
8+
autovacuum_vacuum_threshold = 10000,
9+
autovacuum_analyze_threshold = 10000
10+
);
11+
12+
/* transaction */
13+
-- Add new column to the parent transaction table
14+
ALTER TABLE transaction
15+
ADD COLUMN events JSONB NOT NULL DEFAULT '[]'::JSONB;
16+
17+
-- Change messages column type to JSON in the parent transaction table
18+
ALTER TABLE transaction
19+
ALTER COLUMN messages DROP DEFAULT,
20+
ALTER COLUMN messages TYPE JSON USING messages::JSON,
21+
ALTER COLUMN messages SET DEFAULT '[]'::JSON;
22+
23+
-- Add new column to the child transaction tables
24+
SELECT format(
25+
'ALTER TABLE %I ADD COLUMN events JSONB NOT NULL DEFAULT ''[]''::JSONB;',
26+
relname
27+
)
28+
FROM pg_class
29+
WHERE relname LIKE 'transaction_%' AND relkind = 'r';
30+
31+
-- Change messages column type to JSON in the child transaction tables
32+
SELECT format(
33+
$$
34+
ALTER TABLE %I
35+
ALTER COLUMN messages DROP DEFAULT,
36+
ALTER COLUMN messages TYPE JSON USING messages::JSON,
37+
ALTER COLUMN messages SET DEFAULT '[]'::JSON;
38+
$$,
39+
relname
40+
)
41+
FROM pg_class
42+
WHERE relname LIKE 'transaction_%' AND relkind = 'r';
43+
44+
/* message */
45+
-- Change value column from JSONB to JSON
46+
ALTER TABLE message
47+
ALTER COLUMN value DROP DEFAULT,
48+
ALTER COLUMN value TYPE JSON USING value::JSON;
49+
50+
-- Add foreign key to message_type(type)
51+
ALTER TABLE message
52+
ADD CONSTRAINT message_type_fk
53+
FOREIGN KEY (type) REFERENCES message_type(type);
54+
55+
-- Alter child message tables
56+
DO $$
57+
DECLARE
58+
r RECORD;
59+
BEGIN
60+
FOR r IN
61+
SELECT inhrelid::regclass AS partition_name
62+
FROM pg_inherits
63+
WHERE inhparent = 'message'::regclass
64+
LOOP
65+
RAISE NOTICE 'Altering partition: %', r.partition_name;
66+
67+
-- Add FK constraint on type column
68+
EXECUTE format(
69+
'ALTER TABLE %I
70+
ADD CONSTRAINT %I
71+
FOREIGN KEY (type) REFERENCES message_type(type);',
72+
r.partition_name,
73+
r.partition_name || '_type_fk'
74+
);
75+
END LOOP;
76+
END
77+
$$;
78+
79+
80+
/* new message_by_address function */
81+
DROP FUNCTION IF EXISTS messages_by_address(
82+
TEXT[],
83+
TEXT[],
84+
BIGINT,
85+
BIGINT
86+
);
87+
88+
CREATE FUNCTION messages_by_address(
89+
addresses TEXT[],
90+
types TEXT[],
91+
"limit" BIGINT = 100,
92+
"offset" BIGINT = 0)
93+
RETURNS SETOF message AS
94+
$$
95+
SELECT * FROM message
96+
WHERE (cardinality(types) = 0 OR type = ANY (types))
97+
AND addresses && involved_accounts_addresses
98+
ORDER BY height DESC LIMIT "limit" OFFSET "offset"
99+
$$ LANGUAGE sql STABLE;
100+
101+
/* Create message_by_address function */
102+
CREATE FUNCTION messages_by_type(
103+
types text [],
104+
"limit" bigint DEFAULT 100,
105+
"offset" bigint DEFAULT 0)
106+
RETURNS SETOF message AS
107+
$$
108+
SELECT * FROM message
109+
WHERE (cardinality(types) = 0 OR type = ANY (types))
110+
ORDER BY height DESC LIMIT "limit" OFFSET "offset"
111+
$$ LANGUAGE sql STABLE;
112+
113+
114+
/*######################### staking.sql ##########################*/
115+
/* Add column to table vesting_account */
116+
ALTER TABLE staking_pool
117+
ADD COLUMN unbonding_tokens TEXT NOT NULL DEFAULT '',
118+
ADD COLUMN staked_not_bonded_tokens TEXT NOT NULL DEFAULT '';
119+
120+
/* Drop column from table validator_status */
121+
ALTER TABLE validator_status
122+
DROP COLUMN tombstoned;
123+
124+
/*######################### gov.sql ##########################*/
125+
/* Add column to table gov_params */
126+
ALTER TABLE gov_params
127+
ADD COLUMN params JSONB NOT NULL DEFAULT '[]'::JSONB;
128+
129+
/* Drop columns from table gov_params */
130+
ALTER TABLE gov_params
131+
DROP COLUMN deposit_params,
132+
DROP COLUMN voting_params,
133+
DROP COLUMN tally_params;
134+
135+
/* Drop columns from table proposal */
136+
ALTER TABLE proposal
137+
DROP COLUMN proposal_route;
138+
139+
/* Add column to table proposal */
140+
ALTER TABLE proposal
141+
ADD COLUMN metadata TEXT NOT NULL DEFAULT '';
142+
143+
/* Set default for content column */
144+
ALTER TABLE proposal
145+
ALTER COLUMN content SET DEFAULT '[]'::JSONB;
146+
147+
/* Drop constraint unique_deposit from table proposal_deposit */
148+
ALTER TABLE proposal_deposit
149+
DROP CONSTRAINT unique_deposit;
150+
151+
/* Drop constraint proposal_deposit_height_fkey from table proposal_deposit */
152+
ALTER TABLE proposal_deposit
153+
DROP CONSTRAINT IF EXISTS proposal_deposit_height_fkey;
154+
155+
/* Add columns to table proposal_deposit */
156+
ALTER TABLE proposal_deposit
157+
ADD COLUMN timestamp TIMESTAMP,
158+
ADD COLUMN transaction_hash TEXT NOT NULL DEFAULT '';
159+
160+
/* Add constraint to table proposal_deposit */
161+
ALTER TABLE proposal_deposit
162+
ADD CONSTRAINT unique_deposit UNIQUE (proposal_id, depositor_address, transaction_hash);
163+
164+
/* Drop constraint unique_deposit from table proposal_vote */
165+
ALTER TABLE proposal_vote
166+
DROP CONSTRAINT unique_vote;
167+
168+
/* Drop constraint proposal_deposit_height_fkey from table proposal_deposit */
169+
ALTER TABLE proposal_vote
170+
DROP CONSTRAINT IF EXISTS proposal_vote_height_fkey;
171+
172+
/* Add columns to table proposal_deposit */
173+
ALTER TABLE proposal_vote
174+
ADD COLUMN weight TEXT NOT NULL DEFAULT '',
175+
ADD COLUMN timestamp TIMESTAMP;
176+
177+
/* Add constraint to table proposal_deposit */
178+
ALTER TABLE proposal_vote
179+
ADD CONSTRAINT unique_vote UNIQUE (proposal_id, voter_address, option);
180+
181+
/*######################### upgrade.sql ##########################*/
182+
CREATE TABLE software_upgrade_plan
183+
(
184+
proposal_id INTEGER REFERENCES proposal (id) UNIQUE,
185+
plan_name TEXT NOT NULL,
186+
upgrade_height BIGINT NOT NULL,
187+
info TEXT NOT NULL,
188+
height BIGINT NOT NULL
189+
);
190+
CREATE INDEX software_upgrade_plan_proposal_id_index ON software_upgrade_plan (proposal_id);
191+
CREATE INDEX software_upgrade_plan_height_index ON software_upgrade_plan (height);

0 commit comments

Comments
 (0)