Skip to content

Commit 88974bb

Browse files
smiklosovicsmiklosovic
committed
add export, list, import commands
1 parent 0db9d4d commit 88974bb

11 files changed

+800
-20
lines changed

‎src/java/org/apache/cassandra/db/compression/CompressionDictionary.java‎

Lines changed: 95 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,20 @@ public interface CompressionDictionary extends AutoCloseable
4949
*/
5050
byte[] rawDictionary();
5151

52+
/**
53+
* Get checkum of this dictionary.
54+
*
55+
* @return checksum of this dictionary
56+
*/
57+
int checksum();
58+
59+
/**
60+
* Get size of raw dictionary.
61+
*
62+
* @return size of raw dictionary, in bytes
63+
*/
64+
int size();
65+
5266
/**
5367
* Get the kind of the compression algorithm
5468
*
@@ -126,7 +140,7 @@ static CompressionDictionary deserialize(DataInput input, @Nullable CompressionD
126140
throw new IOException("Compression dictionary checksum does not match. " +
127141
"Expected: " + checksum + "; actual: " + calculatedChecksum);
128142

129-
CompressionDictionary dictionary = kind.createDictionary(dictId, dict);
143+
CompressionDictionary dictionary = kind.createDictionary(dictId, dict, checksum);
130144

131145
// update the dictionary manager if it exists
132146
if (manager != null)
@@ -137,6 +151,24 @@ static CompressionDictionary deserialize(DataInput input, @Nullable CompressionD
137151
return dictionary;
138152
}
139153

154+
static LightweightCompressionDictionary createFromRowLightweight(UntypedResultSet.Row row)
155+
{
156+
String kindStr = row.getString("kind");
157+
long dictId = row.getLong("dict_id");
158+
int checksum = row.getInt("dict_checksum");
159+
int size = row.getInt("dict_length");
160+
161+
try
162+
{
163+
return new LightweightCompressionDictionary(new DictId(CompressionDictionary.Kind.valueOf(kindStr), dictId),
164+
checksum, size);
165+
}
166+
catch (IllegalArgumentException ex)
167+
{
168+
throw new IllegalStateException(kindStr + " compression dictionary is not created for dict id " + dictId);
169+
}
170+
}
171+
140172
static CompressionDictionary createFromRow(UntypedResultSet.Row row)
141173
{
142174
String kindStr = row.getString("kind");
@@ -164,7 +196,7 @@ static CompressionDictionary createFromRow(UntypedResultSet.Row row)
164196
kindStr, dictId, storedChecksum, calculatedChecksum));
165197
}
166198

167-
return kind.createDictionary(new DictId(kind, dictId), dict);
199+
return kind.createDictionary(new DictId(kind, dictId), row.getByteArray("dict"), storedChecksum);
168200
}
169201
catch (IllegalArgumentException ex)
170202
{
@@ -188,9 +220,10 @@ enum Kind
188220
// Order matters: the enum ordinal is serialized
189221
ZSTD
190222
{
191-
public CompressionDictionary createDictionary(DictId dictId, byte[] dict)
223+
@Override
224+
public CompressionDictionary createDictionary(DictId dictId, byte[] dict, int checksum)
192225
{
193-
return new ZstdCompressionDictionary(dictId, dict);
226+
return new ZstdCompressionDictionary(dictId, dict, checksum);
194227
}
195228

196229
@Override
@@ -220,9 +253,10 @@ public ICompressionDictionaryTrainer createTrainer(String keyspaceName,
220253
*
221254
* @param dictId the dictionary identifier
222255
* @param dict the raw dictionary bytes
256+
* @param checksum checksum of this dictionary
223257
* @return a compression dictionary instance
224258
*/
225-
public abstract CompressionDictionary createDictionary(CompressionDictionary.DictId dictId, byte[] dict);
259+
public abstract CompressionDictionary createDictionary(CompressionDictionary.DictId dictId, byte[] dict, int checksum);
226260

227261
/**
228262
* Creates a dictionary compressor for this kind
@@ -281,4 +315,60 @@ public String toString()
281315
'}';
282316
}
283317
}
318+
319+
/**
320+
* The purpose of lightweight dictionary is to not carry the actual dictionary bytes for performance reasons.
321+
* Handy for situations when retrieval from the database does not need to contain dictionary
322+
* or the instatiation of a proper dictionary object is not desirable or unnecessary for other,
323+
* mostly performance-related, reasons.
324+
*/
325+
class LightweightCompressionDictionary implements CompressionDictionary
326+
{
327+
private final DictId dictId;
328+
private final int checksum;
329+
private final int size;
330+
331+
public LightweightCompressionDictionary(DictId dictId, int checksum, int size)
332+
{
333+
this.dictId = dictId;
334+
this.checksum = checksum;
335+
this.size = size;
336+
}
337+
338+
@Override
339+
public DictId dictId()
340+
{
341+
return dictId;
342+
}
343+
344+
@Override
345+
public byte[] rawDictionary()
346+
{
347+
return null;
348+
}
349+
350+
@Override
351+
public int checksum()
352+
{
353+
return checksum;
354+
}
355+
356+
@Override
357+
public int size()
358+
{
359+
return size;
360+
}
361+
362+
@Override
363+
public void serialize(DataOutput out) throws IOException
364+
{
365+
throw new UnsupportedOperationException(LightweightCompressionDictionary.class.getName() + " is not meant to be serialized.");
366+
}
367+
368+
@Override
369+
public void close() throws Exception
370+
{
371+
// intentionally empty
372+
}
373+
}
284374
}

‎src/java/org/apache/cassandra/db/compression/CompressionDictionaryDetailsTabularData.java‎

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.cassandra.db.compression;
20+
21+
import java.util.Arrays;
22+
import javax.management.openmbean.ArrayType;
23+
import javax.management.openmbean.CompositeData;
24+
import javax.management.openmbean.CompositeDataSupport;
25+
import javax.management.openmbean.CompositeType;
26+
import javax.management.openmbean.OpenDataException;
27+
import javax.management.openmbean.OpenType;
28+
import javax.management.openmbean.SimpleType;
29+
import javax.management.openmbean.TabularDataSupport;
30+
import javax.management.openmbean.TabularType;
31+
32+
public class CompressionDictionaryDetailsTabularData
33+
{
34+
public static final String KEYSPACE_NAME = "Keyspace";
35+
public static final String TABLE_NAME = "Table";
36+
public static final String DICT_ID_NAME = "DictId";
37+
public static final String DICT_NAME = "Dict";
38+
public static final String KIND_NAME = "Kind";
39+
public static final String CHECKSUM_NAME = "Checksum";
40+
public static final String SIZE_NAME = "Size";
41+
42+
43+
private static final String[] ITEM_NAMES = new String[]{ KEYSPACE_NAME,
44+
TABLE_NAME,
45+
DICT_ID_NAME,
46+
DICT_NAME,
47+
KIND_NAME,
48+
CHECKSUM_NAME,
49+
SIZE_NAME };
50+
51+
private static final String[] ITEM_DESCS = new String[]{ "keyspace",
52+
"table",
53+
"dictionary_id",
54+
"dictionary_bytes",
55+
"kind",
56+
"checksum",
57+
"size" };
58+
59+
private static final String TYPE_NAME = "DictionaryDetails";
60+
private static final String ROW_DESC = "DictionaryDetails";
61+
private static final OpenType<?>[] ITEM_TYPES;
62+
private static final CompositeType COMPOSITE_TYPE;
63+
public static final TabularType TABULAR_TYPE;
64+
65+
static
66+
{
67+
try
68+
{
69+
ITEM_TYPES = new OpenType[]{ SimpleType.STRING, // keyspace
70+
SimpleType.STRING, // table
71+
SimpleType.LONG, // dict id
72+
new ArrayType<String[]>(SimpleType.BYTE, true), // dict bytes
73+
SimpleType.STRING, // kind
74+
SimpleType.INTEGER, // checksum
75+
SimpleType.INTEGER }; // size of dict bytes
76+
77+
COMPOSITE_TYPE = new CompositeType(TYPE_NAME, ROW_DESC, ITEM_NAMES, ITEM_DESCS, ITEM_TYPES);
78+
TABULAR_TYPE = new TabularType(TYPE_NAME, ROW_DESC, COMPOSITE_TYPE, ITEM_NAMES);
79+
}
80+
catch (OpenDataException e)
81+
{
82+
throw new RuntimeException(e);
83+
}
84+
}
85+
86+
public static void from(String keyspace,
87+
String table,
88+
CompressionDictionary dictionary,
89+
TabularDataSupport result)
90+
{
91+
result.put(from(keyspace, table, dictionary));
92+
}
93+
94+
public static CompositeData from(String keyspace, String table, CompressionDictionary dictionary)
95+
{
96+
try
97+
{
98+
return new CompositeDataSupport(COMPOSITE_TYPE,
99+
ITEM_NAMES,
100+
new Object[]
101+
{
102+
keyspace,
103+
table,
104+
dictionary.dictId().id,
105+
dictionary.rawDictionary(),
106+
dictionary.kind().name(),
107+
dictionary.checksum(),
108+
dictionary.size(),
109+
});
110+
}
111+
catch (OpenDataException e)
112+
{
113+
throw new RuntimeException(e);
114+
}
115+
}
116+
117+
public static CompositeData from(CompressionDictionaryPojo pojo)
118+
{
119+
try
120+
{
121+
return new CompositeDataSupport(COMPOSITE_TYPE,
122+
ITEM_NAMES,
123+
new Object[]
124+
{
125+
pojo.keyspace,
126+
pojo.table,
127+
pojo.dictId,
128+
pojo.dict,
129+
pojo.kind,
130+
pojo.checksum,
131+
pojo.size
132+
});
133+
}
134+
catch (OpenDataException e)
135+
{
136+
throw new RuntimeException(e);
137+
}
138+
}
139+
140+
/**
141+
* Deserializes data to convenience object to work further with.
142+
*
143+
* @param compositeData data to create pojo from
144+
* @return deserialized composite data to convenience object
145+
* @throws IllegalArgumentException if values in deserialized object are invalid.
146+
* @see CompressionDictionaryPojo#validate()
147+
*/
148+
public static CompressionDictionaryPojo from(CompositeData compositeData)
149+
{
150+
String keyspace = (String) compositeData.get(CompressionDictionaryDetailsTabularData.KEYSPACE_NAME);
151+
String table = (String) compositeData.get(CompressionDictionaryDetailsTabularData.TABLE_NAME);
152+
long dictId = (Long) compositeData.get(CompressionDictionaryDetailsTabularData.DICT_ID_NAME);
153+
byte[] dictionaryBytes = (byte[]) compositeData.get(CompressionDictionaryDetailsTabularData.DICT_NAME);
154+
String kind = (String) compositeData.get(CompressionDictionaryDetailsTabularData.KIND_NAME);
155+
int checksum = (Integer) compositeData.get(CompressionDictionaryDetailsTabularData.CHECKSUM_NAME);
156+
int size = (Integer) compositeData.get(CompressionDictionaryDetailsTabularData.SIZE_NAME);
157+
158+
CompressionDictionaryPojo pojo = new CompressionDictionaryPojo();
159+
pojo.keyspace = keyspace;
160+
pojo.table = table;
161+
pojo.dictId = dictId;
162+
pojo.dict = dictionaryBytes;
163+
pojo.kind = kind;
164+
pojo.checksum = checksum;
165+
pojo.size = size;
166+
167+
pojo.validate();
168+
169+
return pojo;
170+
}
171+
172+
public static class CompressionDictionaryPojo
173+
{
174+
public String keyspace;
175+
public String table;
176+
public long dictId;
177+
public byte[] dict;
178+
public String kind;
179+
public int checksum;
180+
public int size;
181+
182+
/**
183+
* Dictionary is valid if, keyspace and table are specified, dictionary id is strictly positive integer,
184+
* dictionary byte array is not nor not empty,
185+
* kind corresponds to {@code Kind}, checksum and size are bigger than 0.
186+
*/
187+
public void validate()
188+
{
189+
if (keyspace == null)
190+
throw new IllegalArgumentException("keyspace not specified");
191+
if (table == null)
192+
throw new IllegalArgumentException("table not specified");
193+
if (dictId <= 0)
194+
throw new IllegalArgumentException("Provided dictionary id is lower than 0, it is '" + dictId + "'.'");
195+
if (dict == null || dict.length == 0)
196+
throw new IllegalArgumentException("Provided dictionary byte array is null or empty.");
197+
if (kind == null)
198+
throw new IllegalArgumentException("Provided kind is null.");
199+
try
200+
{
201+
CompressionDictionary.Kind.valueOf(kind);
202+
}
203+
catch (IllegalArgumentException ex)
204+
{
205+
throw new IllegalArgumentException("There is no such dictionary kind like '" + kind + "'. Available kinds: " + Arrays.asList(CompressionDictionary.Kind.values()));
206+
}
207+
if (checksum <= 0)
208+
throw new IllegalArgumentException("Checksum has to be strictly positive number, it is '" + checksum + "'.");
209+
if (size <= 0)
210+
throw new IllegalArgumentException("Size has to be strictly positive number, it is '" + size + "'.");
211+
}
212+
}
213+
}

‎src/java/org/apache/cassandra/db/compression/CompressionDictionaryEventHandler.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ public void onNewDictionaryAvailable(CompressionDictionary.DictId dictionaryId)
8686
return;
8787
}
8888

89-
CompressionDictionary dictionary = SystemDistributedKeyspace.retrieveCompressionDictionary(keyspaceName, tableName, dictionaryId);
89+
CompressionDictionary dictionary = SystemDistributedKeyspace.retrieveCompressionDictionary(keyspaceName, tableName, dictionaryId.id);
9090
cache.add(dictionary);
9191
}
9292
catch (Exception e)

0 commit comments

Comments
 (0)