@@ -12,26 +12,18 @@ pub async fn update_database_questions(
12
12
let query = QuestionDbQuery :: default ( ) ;
13
13
let query_response = query. post ( client) . await ?;
14
14
let total_questions = query_response. get_total_questions ( ) ;
15
-
16
- let chunk_size = 100 ;
17
- let n_chunks = total_questions / chunk_size;
18
- for i in kdam:: tqdm!( 0 ..n_chunks) {
19
- let skip = i * chunk_size;
20
- let take = chunk_size;
21
- let client_copy = client. clone ( ) ;
22
- let db_client_copy = database_client. clone ( ) ;
23
- let resp = QuestionDbQuery :: new ( take, skip) . post ( & client_copy) . await ?;
24
- let questions = resp. get_questions ( ) ;
25
- Question :: multi_insert ( & db_client_copy, questions) . await ?;
26
- }
27
-
28
- if total_questions % chunk_size != 0 {
29
- let skip = n_chunks * chunk_size;
30
- let take = total_questions - skip;
31
- let client_copy = client. clone ( ) ;
32
- let db_client_copy = database_client. clone ( ) ;
33
- let resp = QuestionDbQuery :: new ( take, skip) . post ( & client_copy) . await ?;
34
- Question :: multi_insert ( & db_client_copy, resp. get_questions ( ) ) . await ?;
15
+ let chunk_size: usize = 100 ;
16
+ let total_range = ( 0 ..total_questions) . collect :: < Vec < _ > > ( ) ;
17
+ for chunk in kdam:: tqdm!( total_range. chunks( chunk_size) ) {
18
+ if let Some ( skip) = chunk. first ( ) {
19
+ let client_copy = client. clone ( ) ;
20
+ let db_client_copy = database_client. clone ( ) ;
21
+ let resp = QuestionDbQuery :: new ( chunk. len ( ) as i32 , * skip)
22
+ . post ( & client_copy)
23
+ . await ?;
24
+ let questions = resp. get_questions ( ) ;
25
+ Question :: multi_insert ( & db_client_copy, questions) . await ?;
26
+ }
35
27
}
36
28
Ok ( ( ) )
37
29
}
@@ -110,3 +102,17 @@ pub async fn async_tasks_executor(
110
102
}
111
103
Ok ( ( ) )
112
104
}
105
+
106
+ #[ cfg( test) ]
107
+ mod tests {
108
+
109
+ #[ test]
110
+ fn test_chunking ( ) {
111
+ let range = ( 0 ..11 ) . collect :: < Vec < _ > > ( ) ;
112
+ for ( chunk, first) in range. chunks ( 2 ) . zip ( [ 0 , 2 , 4 , 6 , 8 ] ) {
113
+ assert_eq ! ( chunk. first( ) . unwrap( ) , & first) ;
114
+ assert_eq ! ( chunk. len( ) , 2 ) ;
115
+ }
116
+ assert_eq ! ( range. chunks( 2 ) . nth( 5 ) , Some ( vec![ 10 ] . as_slice( ) ) ) ;
117
+ }
118
+ }
0 commit comments