fix

Author: vince62s

tensor2tensor/data_generators/generator_utils.py

@@ -338,9 +338,9 @@ def generate():
 
         # Use Tokenizer to count the word occurrences.
         with tf.gfile.GFile(filepath, mode="r") as source_file:
-          file_byte_budget = 1e6 if filepath.endswith("en") else 1e6
+          file_byte_budget = 1e6
           counter = 0
-          countermax = int(source_file.size() / 1e6)
+          countermax = int(source_file.size() / file_byte_budget / 2)
           for line in source_file:
             if counter < countermax:
               counter += 1