New BRNN directory with the source code used to train, validate and evaluate that model.

Author: francisco-ortin

BRNN/configuration.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Configuration file with global constants
+"""
+
+import math
+
+# Data
+DATA_PCT = 0.001
+MODELS_DIR = './models/'
+LOG_DIR = './logs/'
+CSV_DIR = './csvs/'
+PICKLE_FILE_NAMES_7M = {
+    "x_train": "pkl/7_000_000_train.pkl",
+    "y_train": "pkl/7_000_000_train_labels.pkl",
+    "x_val": "pkl/7_000_000_valid.pkl",
+    "y_val": "pkl/7_000_000_valid_labels.pkl",
+    "model": "7_000_000"
+}
+PICKLE_FILE_NAMES_70M = {
+    "x_train": "pkl/70_000_000_train.pkl",
+    "y_train": "pkl/70_000_000_train_labels.pkl",
+    "x_val": "pkl/70_000_000_valid.pkl",
+    "y_val": "pkl/70_000_000_valid_labels.pkl",
+    "model": "70_000_000"
+}
+PICKLE_FILE_NAMES_400M = {
+    "x_train": "pkl/432_180_483_train.pkl",
+    "y_train": "pkl/432_180_483_train_labels.pkl",
+    "x_val": "pkl/1_000_020_valid.pkl",
+    "y_val": "pkl/1_000_020_valid_labels.pkl",
+    "x_test": "pkl/1_000_020_test.pkl",
+    "y_test": "pkl/1_000_020_test_labels.pkl",
+    "model": "400_000_000"
+}
+PICKLE_FILE_NAMES_700M = {
+    "x_train": "pkl/700_000_000_train.pkl",
+    "y_train": "pkl/700_000_000_train_labels.pkl",
+    "x_val": "pkl/700_000_000_valid.pkl",
+    "y_val": "pkl/700_000_000_valid_labels.pkl",
+    "x_test": "pkl/1_000_000_test.pkl",
+    "y_test": "pkl/1_000_000_test_labels.pkl",
+    "model": "700_000_000"
+}
+
+# Input
+VOCABULARY_SIZE = 97  # Size of the vocabulary
+MAX_CHARS_PER_LINE = 40  # Max number of characters per line
+NUMBER_OF_CLASSES = 21  # Number of languages (different targets)
+
+
+# Model hyper-parameters
+EMBEDDING_DIM = value if (value := math.ceil(math.log2(VOCABULARY_SIZE))) % 2 == 0 else value + 1
+EMBEDDING_DIM *= 4
+
+# Programming languages that can be classified
+LANGUAGES = ["Assembly", "C", "C++", "C#", "CSS", "Go", "HTML", "Java", "JavaScript", "Kotlin",
+                   "Matlab", "Perl", "PHP", "Python", "R", "Ruby", "Scala", "SQL", "Swift", "TypeScript",
+                   "Unix Shell"]

BRNN/data.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Module that loads datasets from files
+"""
+
+import pickle
+from typing import Dict
+
+def load_data(file_names:Dict[str,str]):
+    with open(file_names['x_train'], 'rb') as handle:
+        x_train = pickle.load(handle)
+    with open(file_names['y_train'], 'rb') as handle:
+        y_train = pickle.load(handle)
+    with open(file_names['x_val'], 'rb') as handle:
+        x_val = pickle.load(handle)
+    with open(file_names['y_val'], 'rb') as handle:
+        y_val = pickle.load(handle)
+    return (x_train, y_train), (x_val, y_val)
+
+
+def select_first_in_list(iterable, pencentage: float):
+    if pencentage >= 1:
+        return iterable
+    return iterable[:int(len(iterable)*pencentage)]
+
+
+def show_data(x_train, y_train, x_val, y_val):
+    print(f"x_train shape: {x_train.shape}")
+    print(f"y_train shape: {y_train.shape}")
+    print(f"x_val shape: {x_val.shape}")
+    print(f"y_val shape: {y_val.shape}")
+

BRNN/error_analysis.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Performs an error analysis, showing the intances that were missclassified
+"""
+
+import pickle
+import keras
+import numpy as np
+from configuration import PICKLE_FILE_NAMES_400M
+
+MODEL_FILE_NAME = "models/RNN-432180483-batch_size_2048-n_rnn_layers_8-drop_out_0-n_neurons_hidden_dense_layer_classifier_512-n_class_layers_2-learning_rate_0.0001-n_neurons_lstm_out_256-embedding_dim_32-activation_relu-lstm_True-"
+LANGUAGE_LABELS = ["Assembly", "C", "C++", "C#", "CSS", "Go", "HTML", "Java", "JavaScript", "Kotlin",
+                   "Matlab", "Perl", "PHP", "Python", "R", "Ruby", "Scala", "SQL", "Swift", "TypeScript",
+                   "Unix Shell"]
+NUMBER_INSTANCES_TO_PROCESS = 100_000
+
+
+def load_dataset(x_file_name: str, y_file_name: str):
+    print("Loading X for validation...")
+    with open(x_file_name, 'rb') as handle:
+        x_test = pickle.load(handle)
+    print("Loading Y validation...")
+    with open(y_file_name, 'rb') as handle:
+        y_test = pickle.load(handle)
+    # Convert from one-hot to integer values
+    y_test = np.argmax(y_test, axis=1)
+    print(x_test.shape)
+    print(y_test.shape)
+    return x_test, y_test
+
+
+def shuffle_dataset(x_data, y_data):
+    print("Shuffling dataset...")
+    assert len(x_data) == len(y_data)
+    indices = np.random.permutation(len(x_data))
+    return x_data[indices], y_data[indices]
+
+
+def get_erroneous_predictions(y_test, predicted_y, actual_lang: str, predicted_lang: str):
+    assert len(y_test) == len(predicted_y)
+    actual_lang_index, predicted_lang_index = LANGUAGE_LABELS.index(actual_lang), LANGUAGE_LABELS.index(predicted_lang)
+    wrong_classification_indexes = [i for i in range(len(y_test)) if y_test[i] == actual_lang_index and
+                                    np.argmax(predicted_y[i]) == predicted_lang_index]
+    return wrong_classification_indexes
+
+
+def convert_vector_to_code(vector) -> str:
+    return "".join(map(lambda integer: chr(integer - 2 + 32), vector))
+
+
+def show_error_instances(wrong_classification_indexes, x_test):
+    for index in wrong_classification_indexes:
+        print(f'Source code line: "{convert_vector_to_code(x_test[index])}".')
+
+
+def show_miss_classifications(actual_lang: str, predicted_lang: str):
+    # Load dataset
+    x_test, y_test = load_dataset(PICKLE_FILE_NAMES_400M['x_test'], PICKLE_FILE_NAMES_400M['y_test'])
+    x_test, y_test = shuffle_dataset(x_test, y_test)
+    x_test, y_test = x_test[:NUMBER_INSTANCES_TO_PROCESS], y_test[:NUMBER_INSTANCES_TO_PROCESS]
+    # Load model
+    print(f"Loading the model from {MODEL_FILE_NAME} ...")
+    model = keras.models.load_model(MODEL_FILE_NAME)
+    # Predict the languages
+    predicted_y = model.predict(x_test)
+    wrong_classification_indexes = get_erroneous_predictions(y_test, predicted_y, actual_lang, predicted_lang)
+    print(f"Miss classifications of {actual_lang} (actual) as {predicted_lang} (predicted):")
+    show_error_instances(wrong_classification_indexes, x_test)
+
+
+def main():
+    show_miss_classifications("Swift", "Kotlin")
+
+
+if __name__ == "__main__":
+    main()

BRNN/hyperparams.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Class modeling the hyperparameters of neural networks
+"""
+
+class HyperParams:
+
+    def __init__(self, model_name: str, n_individuals: int, batch_size: int,
+                 n_attention_heads: int = None, n_trans_blocks: int = None,
+                 n_rnn_layers: int = None,
+                 n_neurons_hidden_dense_layer_encoder: int = None, drop_out: int = None,
+                 n_neurons_hidden_dense_layer_classifier: int = None, n_class_layers: int = None,
+                 learning_rate: int = None, n_neurons_lstm_out: int = None,
+                 embedding_dim: int = None, activation: str = None, lstm: bool = None):
+        self.properties = dict()
+        self.model_name = model_name
+        self.properties['n_individuals'] = n_individuals
+        self.properties['batch_size'] = batch_size
+        if n_attention_heads is not None:
+            self.properties['n_attention_heads'] = n_attention_heads
+        if n_trans_blocks is not None:
+            self.properties['n_trans_blocks'] = n_trans_blocks
+        if n_rnn_layers is not None:
+            self.properties['n_rnn_layers'] = n_rnn_layers
+        if n_neurons_hidden_dense_layer_encoder is not None:
+            self.properties['n_neurons_hidden_dense_layer_encoder'] = n_neurons_hidden_dense_layer_encoder
+        if drop_out is not None:
+            self.properties['drop_out'] = drop_out
+        if n_neurons_hidden_dense_layer_classifier is not None:
+            self.properties['n_neurons_hidden_dense_layer_classifier'] = n_neurons_hidden_dense_layer_classifier
+        if n_class_layers is not None:
+            self.properties['n_class_layers'] = n_class_layers
+        if learning_rate is not None:
+            self.properties['learning_rate'] = learning_rate
+        if n_neurons_lstm_out is not None:
+            self.properties['n_neurons_lstm_out'] = n_neurons_lstm_out
+        if embedding_dim is not None:
+            self.properties['embedding_dim'] = embedding_dim
+        if activation is not None:
+            self.properties['activation'] = activation
+        if lstm is not None:
+            self.properties['lstm'] = lstm
+
+    def get(self, name: str) -> any:
+        return self.properties[name]
+
+    def set(self, name: str, value: any):
+        self.properties[name] = value
+
+    def long_name(self):
+        result = f"{self.model_name}-{self.properties['n_individuals']}-"
+        for key, value in self.properties.items():
+            if value is not None and key != 'n_individuals':
+                result += f"{key}_{value}-"
+        return result

BRNN/infer-from-params.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Example use of model inference, by restoring it from the saved model
+"""
+import pickle
+import keras
+from configuration import PICKLE_FILE_NAMES_700M, PICKLE_FILE_NAMES_400M
+import os
+from data import select_first_in_list
+from utils import evaluate_model
+
+MODELS_DIR = './models/'
+
+
+def main():
+    file_names = PICKLE_FILE_NAMES_700M
+
+    #model_file_name = MODELS_DIR + file_names['model']
+    model_names = [
+        "RNN-700000000-batch_size_2048-n_rnn_layers_6-drop_out_0-n_neurons_hidden_dense_layer_classifier_512-n_class_layers_2-learning_rate_0.001-n_neurons_lstm_out_256-embedding_dim_32-activation_relu-lstm_True-",
+        "RNN-700000000-batch_size_2048-n_rnn_layers_8-drop_out_0-n_neurons_hidden_dense_layer_classifier_512-n_class_layers_2-learning_rate_0.0005-n_neurons_lstm_out_256-embedding_dim_32-activation_relu-lstm_True-",
+        "RNN-700000000-batch_size_2048-n_rnn_layers_10-drop_out_0-n_neurons_hidden_dense_layer_classifier_512-n_class_layers_2-learning_rate_0.0001-n_neurons_lstm_out_256-embedding_dim_32-activation_relu-lstm_True-"
+        ]
+
+    print("Loading X for validation...")
+    with open(file_names['x_test'], 'rb') as handle:
+        x_val = pickle.load(handle)
+    print("Loading Y validation...")
+    with open(file_names['y_test'], 'rb') as handle:
+        y_val = pickle.load(handle)
+
+    DATA_PCT = 1
+    x_val, y_val = select_first_in_list(x_val, DATA_PCT), select_first_in_list(y_val, DATA_PCT)
+
+    for model_file_name in model_names:
+        model_file_name = MODELS_DIR + model_file_name
+        if not os.path.exists(model_file_name):
+            print(f"File '{model_file_name}' not found.")
+            return
+        print(f"Loading model from '{model_file_name}'...")
+        model = keras.models.load_model(model_file_name)
+        print(f"Evaluating the model {model_file_name}...")
+
+        evaluate_model(model, x_val, y_val)
+
+
+if __name__ == "__main__":
+    main()

BRNN/infer-from-pickle.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Example use of model inference, by restoring it from a serialized model (using pickle)
+"""
+
+import pickle
+from utils import evaluate_model
+import os
+
+MODELS_DIR = './models/'
+PICKLE_FILE_NAMES_7M = {
+    "x_train": "pkl/7_000_000_train.pkl",
+    "y_train": "pkl/7_000_000_train_labels.pkl",
+    "x_val": "pkl/7_000_000_valid.pkl",
+    "y_val": "pkl/7_000_000_valid_labels.pkl",
+}
+
+def get_file_with_highest_accuracy(dir: str) -> str:
+    file_names = os.listdir(dir)
+    file_names = list(filter(lambda file_name: file_name.startswith('accuracy'), file_names))
+    file_names.sort()
+    if len(file_names):
+        return dir + file_names[-1]
+    return None
+
+def main():
+    file_name = get_file_with_highest_accuracy(MODELS_DIR)
+    if not file_name:
+        return
+    print("Loading X for validation...")
+    with open(PICKLE_FILE_NAMES_7M['x_val'], 'rb') as handle:
+        x_val = pickle.load(handle)
+    print("Loading Y validation...")
+    with open(PICKLE_FILE_NAMES_7M['y_val'], 'rb') as handle:
+        y_val = pickle.load(handle)
+    print(f"Loading the model from {file_name} ...")
+    with open(file_name, 'rb') as handle:
+        model = pickle.load(handle)
+    print("Evaluating the model...")
+    evaluate_model(model, x_val, y_val)
+
+
+if __name__ == "__main__":
+    main()