Skip to content

Commit a42049c

Browse files
Johnnykoch02Johnnykoch02
committed
Network Modifications
1 parent 440979e commit a42049c

5 files changed

Lines changed: 70 additions & 36 deletions

File tree

‎README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,5 @@ NOTE: for GPU / CUDA utilization this will not be a tutorial for this.
66

77

88

9+
- linux requires special pyaudio setup
10+

‎data/Models/KeywordModel/Version/0.01‎

8 MB
Binary file not shown.

‎main.py‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,19 @@ def main():
77
# from src.utils import convert_path_to_wav
88
# # convert_path_to_wav("C:\\Users\\John\\Documents\\Sound recordings", 'w4a', 'data/Models/SpeechModel/Training/0/')
99
# # from src.Controllers import AudioController
10+
from src.utils import convert_path_to_wav
11+
# convert_path_to_wav("C:\\Users\\John\\Documents\\Sound recordings", 'w4a', 'data/Models/SpeechModel/Training/0/')
12+
from src.Controllers import AudioController
1013

1114
# aud_cntrl = AudioController.AudioController()
1215
# aud_cntrl.set_stream_window(True)
1316
# aud_cntrl.set_mode_sample()
14-
# aud_cntrl.set_mode_collect_data(10000, os.path.join(os.getcwd(), 'data', 'Models', 'KeywordModel', 'Training', 'Mel_Imgs', '0'), None)
17+
# aud_cntrl.set_mode_collect_data(100, os.path.join(os.getcwd(), 'data', 'Models', 'KeywordModel', 'Training', 'Mel_Imgs', '1'), None)
1518

16-
# from src.Gwen.AISystem.Networks import KeywordAudioModel
19+
from src.Gwen.AISystem.Networks import KeywordAudioModel
1720

18-
# model = KeywordAudioModel()
19-
# model.train(10, model.load_in_data())
21+
model = KeywordAudioModel()
22+
model.train(1000, model.load_in_data())
2023

2124
# aud_cntrl = AudioController.AudioController()
2225
# aud_cntrl.set_stream_window(True)
@@ -33,6 +36,7 @@ def main():
3336

3437
n.quit()
3538

39+
3640
if __name__ == '__main__':
3741
main()
3842

‎src/Controllers/AudioController.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ def __init__(self, keyword='', stream_visible=True):
3737
self.new_user_Wait_flag = False
3838

3939
'''Data Collection Stuff'''
40-
self._data_path = None
40+
self._data_path = os.path.join(os.getcwd(), 'data', 'Models', 'KeywordModel', 'Training', 'Mel_Imgs', '0')
4141
self._step_number = 0
4242
self._stream_window_visble = stream_visible
4343
self._sample_episode = 100
@@ -89,7 +89,7 @@ def collect_data(self, num_samples, path, sample_episode=100):
8989
self._current_stream_img = audio_img
9090

9191
np.save(os.path.join(path, f'{index}.npy'), arr=audio_img, allow_pickle=True)
92-
t.sleep(1.0)
92+
# t.sleep(1.0)
9393
except sr.UnknownValueError:
9494
print("Could not understand audio")
9595
except sr.RequestError as e:

‎src/Gwen/AISystem/Networks.py‎

Lines changed: 58 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,12 @@
33
from src.Gwen.AISystem.preprocess import *
44
import subprocess
55
import numpy as np
6-
import librosa
7-
from sklearn.model_selection import train_test_split
8-
from keras.utils import to_categorical
96
import matplotlib.pyplot as plt
107
import torch as th
118
import torch.nn as nn
9+
from sklearn.preprocessing import Normalizer
1210
from torch.utils.data import DataLoader
11+
from keras.preprocessing.image import ImageDataGenerator
1312

1413
class Dataset(th.utils.data.Dataset):
1514
def __init__(self, data, labels):
@@ -36,61 +35,88 @@ class KeywordAudioModel(nn.Module):
3635
https://arxiv.org/pdf/2005.06720v2.pdf
3736
The structure of the network is designed to implement the following Paper.
3837
'''
39-
def __init__(self, input_shape=(1, 64, 72), lr=0.015, VERSION="0.01"):
38+
def __init__(self, input_shape=(1, 64, 72), lr=0.08, VERSION="0.01"):
4039
super(KeywordAudioModel, self).__init__()
4140
'''3x64x48'''
4241
self.batch_size = 32
4342
self._ConvolutionalLayers = nn.Sequential(
4443
nn.Conv2d(in_channels=1, out_channels=5, kernel_size=(5,7), padding=5, bias=True), # output (5 x 78 x 68)
44+
nn.BatchNorm2d(5),
4545
nn.LeakyReLU(),
46-
nn.Conv2d(in_channels=5, out_channels=10, kernel_size=(4,4), stride=2), # Output (10 x 38 x 33)
46+
nn.Conv2d(in_channels=5, out_channels=10, kernel_size=(4,4)), # Output (10 x 38 x 33)
47+
nn.BatchNorm2d(10),
4748
nn.LeakyReLU(),
48-
nn.Conv2d(in_channels=10, out_channels=20, kernel_size=(6,5), stride=4), # Output (20 x 9 x 8)
49+
nn.MaxPool2d(2, 2),
50+
nn.Conv2d(in_channels=10, out_channels=20, kernel_size=(2,2), stride=2), # Output (20 x 9 x 8)
51+
nn.BatchNorm2d(20),
4952
nn.LeakyReLU(),
53+
nn.MaxPool2d(2, 2),
5054
nn.Flatten(),
51-
) # Output 2600
55+
) # Output 1440
5256

53-
self._ReccurrentLayers = {
54-
'input_layer': nn.Linear(1440, 2200),
55-
# 'reccurrent_layer': nn.Linear(2800, 200)
56-
}
57+
# self._ReccurrentLayers = {
58+
# 'input_layer': nn.Linear(1440, 2200),
59+
# # 'reccurrent_layer': nn.Linear(2800, 200)
60+
# }
5761

5862
# self._ReccurrentHidden = self._init_reccurrent()
5963
self.Data_Path = os.path.join(os.getcwd(), "data","Models","KeywordModel","Training")
6064
self._LinearLayers = nn.Sequential(
61-
nn.Linear(2200, 900, bias=True),
65+
nn.Linear(1440, 900, bias=True),
6266
nn.LeakyReLU(),
63-
nn.Linear(900, 120),
64-
nn.LeakyReLU(),
65-
nn.Linear(120, 2),
66-
nn.Softmax()
67+
nn.Linear(900, 2),
68+
nn.Softmax(dim=1)
6769
)
6870

71+
self.device = 'cuda' if th.cuda.is_available() else 'cpu'
72+
self.to(self.device)
73+
6974
self._loss_func = nn.CrossEntropyLoss()
70-
self._optimizer = th.optim.Adam(self.parameters(), lr=lr)
75+
self._optimizer = th.optim.RMSprop(self.parameters(), lr=lr)
7176

7277
self.num_classes = 2
7378
self.VERSION = VERSION
79+
80+
self._training_data = self.load_in_data(fit=True)
81+
self.normalizer = ImageDataGenerator(featurewise_center=True, featurewise_std_normalization=True) #Issues with normalization: Need to convert for Pytorch compatability
82+
self.normalizer.fit(self._training_data)
83+
84+
7485
# def _init_reccurrent(self):
7586
# return th.zeros(shape=(1,200))
7687
def forward(self, x):
77-
return self._LinearLayers(self._ReccurrentLayers['input_layer'](self._ConvolutionalLayers(x)))
88+
x = x.to(self.device)
89+
return self._LinearLayers(self._ConvolutionalLayers(x))
7890

7991
def preload_data(self):
8092
save_data_to_array(path=self.Data_Path)
8193

82-
def load_in_data(self):
94+
def load_in_data(self, fit=False):
8395
data = []
84-
for dir in os.listdir(os.path.join(self.Data_Path, "Mel_Imgs")):
85-
for file in os.listdir(os.path.join(self.Data_Path, "Mel_Imgs", dir)):
86-
data.append(
87-
[np.array([np.load(os.path.join(self.Data_Path, "Mel_Imgs", dir, file), allow_pickle=True)]), int(dir)]
88-
)
89-
# print(data[-1][0].shape)
90-
data = np.array(data)
91-
92-
return DataLoader(Dataset(data[:,0], data[:,1]), batch_size=self.batch_size, shuffle=True)
96+
if not fit:
97+
for dir in os.listdir(os.path.join(self.Data_Path, "Mel_Imgs")):
98+
for i, file in enumerate(os.listdir(os.path.join(self.Data_Path, "Mel_Imgs", dir))):
99+
if i > 1000: # Temp Fix for large num of files, get rid of later
100+
break
101+
img = np.load(os.path.join(self.Data_Path, "Mel_Imgs", dir, file), allow_pickle=True)
102+
data.append(
103+
[np.array(self.normalizer.standardize([img])), int(dir)]
104+
)
105+
data = np.array(data)
106+
return DataLoader(Dataset(data[:,0], data[:,1]), batch_size=self.batch_size, shuffle=True)
107+
else:
108+
for dir in os.listdir(os.path.join(self.Data_Path, "Mel_Imgs")):
109+
for file in os.listdir(os.path.join(self.Data_Path, "Mel_Imgs", dir)):
110+
img = np.load(os.path.join(self.Data_Path, "Mel_Imgs", dir, file), allow_pickle=True)
111+
data.append(
112+
[img]
113+
)
114+
return np.array(data)
93115

116+
def predict(self, x):
117+
x = th.from_numpy(self.normalizer.standardize(x)).to(self.device)
118+
return th.argmax(self.forward(x)).item()
119+
94120

95121
def train(self, num_epochs, data_loader: DataLoader, save_location=None):
96122
from torch.autograd import Variable
@@ -104,8 +130,10 @@ def train(self, num_epochs, data_loader: DataLoader, save_location=None):
104130
self._optimizer.zero_grad()
105131
# mel_imgs = mel_imgs.to("cuda") if th.cuda.is_available() else mel_imgs
106132
mel_imgs = mel_imgs.float()
107-
output = self.forward(mel_imgs)
108-
loss = self._loss_func(output.squeeze(), output)
133+
out = self.forward(mel_imgs)
134+
output = th.nn.functional.one_hot(output, self.num_classes).float().to(self.device)
135+
136+
loss = self._loss_func(out, output)
109137
loss.backward()
110138
self._optimizer.step()
111139
epoch_loss += loss.item()

0 commit comments

Comments
 (0)