diff --git a/docker-compose.yml b/docker-compose.yml index 80eb75e..3aa1857 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,7 @@ services: - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} volumes: - ./src:/opt/src + command: ./entryPoint.sh depends_on: - redis @@ -29,6 +30,25 @@ services: - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} command: ['python', 'app/models/InceptionV3/inception_inference_server.py'] + volumes: + - ./src:/opt/src + + networks: + - michaniki + depends_on: + - michaniki_client + - redis + + sentiment_inference_server: + build: + context: ./src + dockerfile: Dockerfile-sentiment + environment: + - REDIS_URL="redis://redis" + - REDIS_PORT=6379 + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + command: ['python', 'app/models/SentimentV1/sentiment_infer_server.py'] volumes: - ./src:/opt/src networks: @@ -36,7 +56,7 @@ services: depends_on: - michaniki_client - redis - + celery_worker: build: ./src command: ['celery', '-A', 'app.celeryapp:michaniki_celery_app', 'worker', '-l', 'info'] @@ -54,14 +74,13 @@ services: - DB_PASSWORD=michaniki - DB_NAME=michanikidb - BROKER_URL=redis://redis:6379/0 - + redis: image: redis:4.0.5-alpine command: ["redis-server", "--appendonly", "yes"] hostname: redis networks: - michaniki - + networks: michaniki: - \ No newline at end of file diff --git a/src/Dockerfile-sentiment b/src/Dockerfile-sentiment new file mode 100644 index 0000000..5b5c09a --- /dev/null +++ b/src/Dockerfile-sentiment @@ -0,0 +1,23 @@ +FROM continuumio/miniconda3:4.5.12 + +# utils +RUN apt-get update && apt-get install -y --no-install-recommends apt-utils + +RUN conda install gxx_linux-64 + +RUN conda install python=3.6 + +RUN apt-get install -y --force-yes default-libmysqlclient-dev mysql-client build-essential + +# Grab requirements.txt. +COPY requirementssenti.txt /tmp/requirementssenti.txt + +# Install dependencies +RUN pip install -qr /tmp/requirementssenti.txt + +# create a user for web server +RUN adduser --disabled-password --gecos "" foo + +COPY ./ /opt/src + +WORKDIR /opt/src diff --git a/src/app/apis/InceptionV3/inceptionV3.py b/src/app/apis/InceptionV3/inceptionV3.py index 8c2caa4..32dddb1 100644 --- a/src/app/apis/InceptionV3/inceptionV3.py +++ b/src/app/apis/InceptionV3/inceptionV3.py @@ -48,12 +48,12 @@ def label(): s3_bucket_name = request.form.get('s3_bucket_name') s3_bucket_prefix = request.form.get('s3_bucket_prefix') model_name = request.form.get('model_name') - + # load image from s3 image_data_path = API_helpers.download_a_dir_from_s3(s3_bucket_name, - s3_bucket_prefix, + s3_bucket_prefix, local_path = TEMP_FOLDER) - + # for each images in the folder # supports .png and .jpg all_image_ids = [] @@ -62,28 +62,28 @@ def label(): for each_image in glob.glob(image_data_path + "/*.*"): iamge_name = each_image.split('/')[-1] this_img = image.load_img(each_image, target_size = (299, 299)) - + # image pre-processing x = np.expand_dims(image.img_to_array(this_img), axis=0) x = preprocess_input(x) x = x.copy(order="C") - + # encode x = API_helpers.base64_encode_image(x) # create a image id this_id = str(uuid.uuid4()) all_image_ids.append((this_id, iamge_name)) d = {"id": this_id, "image": x, "model_name": model_name} - + # push to the redis queue db.rpush(INCEPTIONV3_IMAGE_QUEUE, json.dumps(d)) - + all_pred = [] while all_image_ids: # pop the first one from the queue this_id, this_image_name = all_image_ids.pop(0) this_pred = {} - + while True: # check if the response has been returned output = db.get(this_id) @@ -92,27 +92,27 @@ def label(): this_pred["image name"] = this_image_name output = output.decode('utf-8') this_pred["prediction"] = json.loads(output) - + db.delete(this_id) break else: time.sleep(CLIENT_SLEEP) - + all_pred.append(this_pred) - + # remove the temp folder shutil.rmtree(image_data_path, ignore_errors=True) - + return jsonify({ "data": all_pred - }) - + }) + @blueprint.route('/retrain', methods=['POST']) def retrain(): """ pick up a pre-trained model resume training using more data - + @args: train_bucket_url: URL pointing to the folder for training data on S3 @args: model_name: the name of the model want to be retraiend, the folder must be exsit """ @@ -120,15 +120,15 @@ def retrain(): s3_bucket_prefix = request.form.get('train_bucket_prefix') nb_epoch = request.form.get('nb_epoch') batch_size = request.form.get('batch_size') - + model_name = s3_bucket_prefix.split('/')[-1] local_data_path = os.path.join('./tmp') - + # create a celer task id this_id = celery.uuid() # download the folder in the url # return the path of the image files - async_retrain.apply_async((model_name, + async_retrain.apply_async((model_name, local_data_path, s3_bucket_name, s3_bucket_prefix, @@ -140,46 +140,46 @@ def retrain(): "task_id": this_id, "status": "Retraining and Fine-Tuning are Initiated" }), 200 - + @blueprint.route('/transfer', methods=['POST']) def init_new_model(): """ init a new model based on InceptionV3 that can predict picture for new classes. - + @args: train_bucket_url: URL pointing to the folder for training data on S3 """ # need to load the base model here s3_bucket_name = request.form.get('train_bucket_name') s3_bucket_prefix = request.form.get('train_bucket_prefix') model_name = s3_bucket_prefix.split('/')[-1] - + # generate a celery task id this_id = celery.uuid() # download the folder in the url # kick off the transfer learning thing here - async_transfer.apply_async((model_name, + async_transfer.apply_async((model_name, s3_bucket_name, s3_bucket_prefix, this_id), task_id=this_id) - + return jsonify({ "task_id": this_id, "status": "Transfer Learning and Fine-Tuning are Initiated" }), 200 - + @blueprint.route('/predict', methods=['POST']) def run_inceptionV3(): """ - Run the pre-trained base Inception V3 model + Run the pre-trained base Inception V3 model and send image to queue - - Listening user submitted images and + + Listening user submitted images and stack them in a Redis queue """ data = {"success": False} - + # load model name model_name = request.form.get('model_name') @@ -189,41 +189,33 @@ def run_inceptionV3(): x = np.expand_dims(image.img_to_array(img), axis=0) x = preprocess_input(x) x = x.copy(order="C") - + # encode x = API_helpers.base64_encode_image(x) # create a image id this_id = str(uuid.uuid4()) - + d = {"id": this_id, "image": x, "model_name": model_name} - + # push to the redis queue db.rpush(INCEPTIONV3_IMAGE_QUEUE, json.dumps(d)) while True: # check if the response has been returned output = db.get(this_id) - + if output is not None: output = output.decode('utf-8') data["prediction"] = json.loads(output) - + db.delete(this_id) break else: # print "* Waiting for the Inference Server..." time.sleep(CLIENT_SLEEP) - + data['success'] = True - + return jsonify({ "data": data }), 200 - - - - - - - - \ No newline at end of file diff --git a/src/app/apis/SentimentV1/__init__.py b/src/app/apis/SentimentV1/__init__.py new file mode 100644 index 0000000..1d6da5b --- /dev/null +++ b/src/app/apis/SentimentV1/__init__.py @@ -0,0 +1,6 @@ +''' +Created on Jan 23, 2019 + +@author: manu +''' +from sentimentV1 import * diff --git a/src/app/apis/SentimentV1/sentimentV1.py b/src/app/apis/SentimentV1/sentimentV1.py new file mode 100644 index 0000000..5c6edc7 --- /dev/null +++ b/src/app/apis/SentimentV1/sentimentV1.py @@ -0,0 +1,70 @@ +''' +Created on Jan 22, 2019 + +Web service for Sentiment Analysis + +@author: manu +''' +import uuid +import json +import time + +# flask +from flask import jsonify +from flask import Blueprint, request + +from app import app +from app import db +import logging +# michaniki app +from ...tasks import * + +# temp folder save image files downloaded from S3 +TEMP_FOLDER = os.path.join('./tmp') + +blueprint = Blueprint('sentimentV1', __name__) + +@blueprint.route('/predict', methods=['POST']) +def pred_sentiment(): + """ + Run the pre-trained base Sentiment analysis model + and send sentence to queue + + Listening user submitted sentences and + stack them in a Redis queue + """ + + logging.info("Inside pred_Sentence") + data = {"success": False} + + model_name = 'base' + + message = request.form.get('textv') + print "Received message:{}".format(message) + #sentence = Sentence(message) + + # create a image id + this_id = str(uuid.uuid4()) + + d = {"id": this_id, "text": message, "model_name": model_name} + + # push to the redis queue + db.rpush(SENTIMENT_TEXT_QUEUE, json.dumps(d)) + + while True: + # check if the response has been returned + output = db.get(this_id) + if output is not None: + output = output.decode('utf-8') + data["prediction"] = json.loads(output) + + db.delete(this_id) + break + else: + #print "* Waiting for the Sentiment Inference Server..." + time.sleep(CLIENT_SLEEP) + + data['success'] = True + return jsonify({ + "data": data + }), 200 diff --git a/src/app/apis/apis.py b/src/app/apis/apis.py index 59ddc2b..08dcb84 100644 --- a/src/app/apis/apis.py +++ b/src/app/apis/apis.py @@ -9,6 +9,7 @@ from .mnist import blueprint as mnist_blueprint from .InceptionV3 import blueprint as incept_blueprint +from .SentimentV1 import blueprint as sentiment_blueprint from .tasks import blueprint as tasks_blueprint from app import app @@ -16,6 +17,7 @@ app.register_blueprint(mnist_blueprint, url_prefix = '/mnist') app.register_blueprint(incept_blueprint, url_prefix = '/inceptionV3') app.register_blueprint(tasks_blueprint, url_prefix='/tasks') +app.register_blueprint(sentiment_blueprint, url_prefix = '/sentimentV1') @app.route('/') def index(): @@ -24,5 +26,3 @@ def index(): @app.route('/add') def add_a(): res = add.delay(3, 4) - - diff --git a/src/app/models/InceptionV3/inceptionV3_transfer_retraining.py b/src/app/models/InceptionV3/inceptionV3_transfer_retraining.py index f65c638..76d6416 100644 --- a/src/app/models/InceptionV3/inceptionV3_transfer_retraining.py +++ b/src/app/models/InceptionV3/inceptionV3_transfer_retraining.py @@ -5,7 +5,7 @@ ''' import os import glob - +import logging import keras from keras.models import Model @@ -24,11 +24,11 @@ class InceptionRetrainer: def __init__(self, model_name): self.model_name = model_name - - def retrain(self, - this_model, - local_dir, - nb_epoch, + + def retrain(self, + this_model, + local_dir, + nb_epoch, batch_size): """ retrain the model @@ -36,35 +36,35 @@ def retrain(self, # load the training data train_dir = os.path.join(local_dir, "train") val_dir = os.path.join(local_dir, "val") - + # set up parameters nb_train_samples = self.__get_nb_files(train_dir) nb_classes = len(glob.glob(train_dir + "/*")) nb_val_samples = self.__get_nb_files(val_dir) nb_epoch = int(nb_epoch) batch_size = int(batch_size) - + # set up image data train_datagen = ImageDataGenerator( preprocessing_function = preprocess_input ) - + val_datagen = ImageDataGenerator( preprocessing_function=preprocess_input ) - + # generator train_generator = train_datagen.flow_from_directory( train_dir, target_size=(299, 299), batch_size=batch_size) - + validation_generator = val_datagen.flow_from_directory( val_dir, target_size=(299, 299), batch_size=batch_size, ) - + # retrain the model retrain_history = this_model.fit_generator(train_generator, nb_epoch=nb_epoch, @@ -75,9 +75,10 @@ def retrain(self, verbose=1) return this_model, retrain_history - + def __get_nb_files(self, directory): """Get number of files by searching local dir recursively""" + logging.info("Inside __get_nb_files") if not os.path.exists(directory): return 0 cnt = 0 @@ -85,7 +86,7 @@ def __get_nb_files(self, directory): for dr in dirs: cnt += len(glob.glob(os.path.join(r, dr + "/*"))) return cnt - + class InceptionTransferLeaner: def __init__(self, model_name): self.model_name = model_name @@ -94,13 +95,14 @@ def __init__(self, model_name): try: print "* Transfer: Loading Topless Model..." self.topless_model = load_model(INCEPTIONV3_TOPLESS_MODEL_PATH) + except IOError: # load model from keras print "* Transfer: Loading Topless Model from Keras..." - self.topless_model = InceptionV3(include_top=False, + self.topless_model = InceptionV3(include_top=False, weights='imagenet', input_shape=(299, 299, 3)) - + self.new_model = None # init the new model def transfer_model(self, local_dir, @@ -110,48 +112,55 @@ def transfer_model(self, local_dir, transfer the topless InceptionV3 model to classify new classes """ + print "Inside Transfer model" + + train_dir = os.path.join(local_dir, "train") val_dir = os.path.join(local_dir, "val") + + # set up parameters nb_train_samples = self.__get_nb_files(train_dir) nb_classes = len(glob.glob(train_dir + "/*")) nb_val_samples = self.__get_nb_files(val_dir) nb_epoch = int(nb_epoch) batch_size = int(batch_size) - + + print "nb_val_samples:{}".format(nb_val_samples) + # data prep train_datagen = ImageDataGenerator( preprocessing_function = preprocess_input ) - + val_datagen = ImageDataGenerator( preprocessing_function=preprocess_input ) - + # generator train_generator = train_datagen.flow_from_directory( train_dir, target_size=(299, 299), batch_size=batch_size) - + validation_generator = val_datagen.flow_from_directory( val_dir, target_size=(299, 299), batch_size=batch_size, ) - + # get the class and label name, reverse key and value pair classes_label_dict = train_generator.class_indices classes_label_dict = {v: k for k, v in classes_label_dict.iteritems()} - + # add a new top layer base on the user data - self.new_model = self.__add_new_last_layer(self.topless_model, nb_classes) - + self.new_model = self.__add_new_last_layer(self.topless_model, nb_classes) + # set up transfer learning model - self.__setup_to_transfer_learn(model=self.new_model, + self.__setup_to_transfer_learn(model=self.new_model, base_model=self.topless_model) - + print "* Transfer: Added a New Last Layer... Starting Transfer Learning..." # train the new model for few epoch # TO DO: @@ -163,10 +172,10 @@ def transfer_model(self, local_dir, nb_val_samples=nb_val_samples//batch_size, class_weight='auto', verbose=2) - + # set up fine-tuning model self.__setup_to_finetune(self.new_model, nb_layer_to_freeze=10) - + print "* Transfer: Starting Fine-Tuning..." # train the new model again to fine-tune it history_ft = self.new_model.fit_generator(train_generator, @@ -179,7 +188,7 @@ def transfer_model(self, local_dir, # return the model return self.new_model, classes_label_dict, history_ft - + def __setup_to_finetune(self, model, nb_layer_to_freeze): """ Freeze the bottom NB_IV3_LAYERS and retrain the remaining top layers. @@ -198,7 +207,7 @@ def __setup_to_transfer_learn(self, model, base_model): for layer in base_model.layers: layer.trainable = False model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) - + def __add_new_last_layer(self, topless_model, nb_classes): """ add the last layer to the topless model @@ -209,9 +218,11 @@ def __add_new_last_layer(self, topless_model, nb_classes): predictions = Dense(nb_classes, activation='softmax')(x) #new softmax layer model = Model(input=topless_model.input, output=predictions) return model - + def __get_nb_files(self, directory): """Get number of files by searching local dir recursively""" + logging.info("Inside __get_nb_files") + logging.info("dir->%s",directory) if not os.path.exists(directory): return 0 cnt = 0 diff --git a/src/app/models/InceptionV3/inception_inference_server.py b/src/app/models/InceptionV3/inception_inference_server.py index c118c5b..bb191bf 100644 --- a/src/app/models/InceptionV3/inception_inference_server.py +++ b/src/app/models/InceptionV3/inception_inference_server.py @@ -31,15 +31,15 @@ class inceptionV3_infernece_server: def __init__(self): # pre-load some models here on start self.loaded_models = {} - + def run_inceptionV3_infernece_server(self): ''' run the inference server for Inception V3 - - Pull image from the Redis, decode + + Pull image from the Redis, decode send to the model, predict return the response to the redis - + Images are tracked using is Image IDs ''' while True: @@ -47,15 +47,15 @@ def run_inceptionV3_infernece_server(self): imageIDs = defaultdict(list) batch_for_each_model = defaultdict(partial(np.ndarray, 0)) num_pic = 0 - + for q in queue: q = json.loads(q.decode("utf-8")) - - # decode image - this_image = INV3_helpers.base64_decode_image(q['image'], + + # decode image + this_image = INV3_helpers.base64_decode_image(q['image'], settings.IMAGE_TYPE, shape = settings.IMAGE_SHAPE) - + model_to_go = str(q['model_name']) # stack up the image to the batch # for each model @@ -66,20 +66,20 @@ def run_inceptionV3_infernece_server(self): # vstack on it batch_for_each_model[model_to_go] = np.vstack([batch_for_each_model[model_to_go], this_image]) - + # add the image id imageIDs[model_to_go].append(q['id']) num_pic += 1 - + # if there is any images in the batch if imageIDs: print "* Predicting for {} of Models".format(len(imageIDs.keys())) print "* Number of Picture: {}".format(num_pic) - + # loop over each model and predict their batch for each_model_name, each_batch in batch_for_each_model.iteritems(): this_ids = imageIDs[each_model_name] # these are the ids for the batch for this model - + # load model here # check the model if already exsit if each_model_name in self.loaded_models.keys(): @@ -91,39 +91,37 @@ def run_inceptionV3_infernece_server(self): model = load_model(os.path.join(settings.InceptionV3_MODEL_PATH, each_model_name, each_model_name+'.h5')) self.loaded_models[each_model_name] = model# save the model instance print "* {} Loaded and Saved in Mem.".format(each_model_name) - + # start predicting preds = model.predict(each_batch) - + # TO DO: # Decode prediction to get the class label results = INV3_helpers.decode_pred_to_label(preds, each_model_name, num_return = settings.NUM_LABEL_TO_RETURN) - + # loop ever each image in the batch for (each_id, each_result) in zip(this_ids, results): this_output = [] - + # generate probability of top classes for (label, prob) in each_result: r = {"label": label, "probability": float(prob)} - + this_output.append(r) - + # add this result to the queue # indexed by image id db.set(each_id, json.dumps(this_output)) print "* Prediction for {} Sent Back!".format(each_model_name) - + # delete this image batch from the queue # to save space db.ltrim(settings.IMAGE_QUEUE, len(imageIDs), -1) - + # sleep and wait time.sleep(settings.SERVER_SLEEP) - + if __name__ == "__main__": this_server = inceptionV3_infernece_server() this_server.run_inceptionV3_infernece_server() - - \ No newline at end of file diff --git a/src/app/models/InceptionV3/settings.py b/src/app/models/InceptionV3/settings.py index 9782495..7e92bca 100644 --- a/src/app/models/InceptionV3/settings.py +++ b/src/app/models/InceptionV3/settings.py @@ -7,4 +7,4 @@ IMAGE_SHAPE = (1, 299, 299, 3) FC_SIZE = 1024 SERVER_SLEEP = 0.5 -NUM_LABEL_TO_RETURN = 5 \ No newline at end of file +NUM_LABEL_TO_RETURN = 5 diff --git a/src/app/models/SentimentV1/__init__.py b/src/app/models/SentimentV1/__init__.py new file mode 100644 index 0000000..c0174b4 --- /dev/null +++ b/src/app/models/SentimentV1/__init__.py @@ -0,0 +1 @@ +from .sentiment_infer_server import * diff --git a/src/app/models/SentimentV1/sentiment_infer_server.py b/src/app/models/SentimentV1/sentiment_infer_server.py new file mode 100644 index 0000000..cd654b5 --- /dev/null +++ b/src/app/models/SentimentV1/sentiment_infer_server.py @@ -0,0 +1,74 @@ +''' +Created on Jan 22, 2019 + +@author: manu +''' +import os +import redis +import time +import json +from textblob import TextBlob +import logging +from collections import defaultdict + +#helpers +import settings + +pool = redis.ConnectionPool(host='redis', port=6379, db=0) +db = redis.Redis(connection_pool=pool) + +class sentimentV1_inference_server: + def __init__(self): + # pre-load some models here on start + self.loaded_models = {} + + def run_sentimentV1_infernece_server(self): + ''' + run the inference server for Sentiment Analysis + + Pull sentence from the Redis, predict + return the response to the redis + + Sentecnes are tracked using is their id + ''' + logging.info("Sentiment Inference Server running") + while True: + queue = db.lrange(settings.TEXT_QUEUE, 0, settings.BATCH_SIZE) #Is this queue different from the Queue in API path + textIDs = defaultdict(list) #dict to hold sentence and id for a model type + text_list=[] + sent_list = [] + num_text=0 + for q in queue: + q = json.loads(q.decode("utf-8")) + + model_name = str(q['model_name']) + id = q['id'] + sentence = q['text'] + logging.info("Sentence in server:%s", sentence) + + text_list.append({"model_name":model_name,"id":id, "text":sentence}) + textIDs[model_name].append(q['id']) + num_text += 1 + + sent_list.append(sentence) + + + if textIDs: + print("* Predicting for {} of Models".format(len(textIDs.keys()))) + print("* Number of Sentences: {}".format(num_text)) + + r = {"positive":0.5, "negative":0.5} + for t in text_list: + logging.info("Text is:%s",t["text"]) + preds = TextBlob(t["text"]) + res = {"polarity":preds.sentiment.polarity,"subjectvity":preds.sentiment.subjectivity} + db.set(t["id"], json.dumps(res)) + + db.ltrim(settings.TEXT_QUEUE, len(textIDs), -1) + + # sleep and wait + time.sleep(settings.SERVER_SLEEP) + +if __name__ == "__main__": + this_server = sentimentV1_inference_server() + this_server.run_sentimentV1_infernece_server() diff --git a/src/app/models/SentimentV1/settings.py b/src/app/models/SentimentV1/settings.py new file mode 100644 index 0000000..c26bdd9 --- /dev/null +++ b/src/app/models/SentimentV1/settings.py @@ -0,0 +1,6 @@ +import os + +BATCH_SIZE = 2 +SERVER_SLEEP = 0.5 +NUM_LABEL_TO_RETURN = 2 +TEXT_QUEUE = 'sentimentV1_text_queue' diff --git a/src/app/tasks.py b/src/app/tasks.py index 009fd06..49ba66d 100644 --- a/src/app/tasks.py +++ b/src/app/tasks.py @@ -19,6 +19,9 @@ INV3_TRANSFER_BATCH_SIZE = app.config['INV3_TRANSFER_BATCH_SIZE'] INCEPTIONV3_IMAGE_QUEUE = app.config['INCEPTIONV3_IMAGE_QUEUE'] INCEPTIONV3_TOPLESS_MODEL_PATH = app.config['INCEPTIONV3_TOPLESS_MODEL_PATH'] + +SENTIMENT_TEXT_QUEUE = app.config['SENTIMENT_TEXT_QUEUE'] #Added by MS on 22-Jan-2019 + TEMP_FOLDER = os.path.join('./tmp') @michaniki_celery_app.task() @@ -31,31 +34,31 @@ def async_retrain(model_name, """ retrain model resume training - """ - # download image data to local + """ + # download image data to local image_data_path = API_helpers.download_a_dir_from_s3(s3_bucket_name, - s3_bucket_prefix, + s3_bucket_prefix, local_path = TEMP_FOLDER) try: this_model_path = os.path.join("app", "models", "InceptionV3", model_name, model_name + ".h5") # load the model this_model = load_model(this_model_path) - + this_retrainer = inceptionV3_transfer_retraining.InceptionRetrainer(model_name) - + # return the retraiend new model new_model, history = this_retrainer.retrain(this_model, - image_data_path, - nb_epoch, + image_data_path, + nb_epoch, batch_size) - + print "* Celery Transfer: Retrained Model Saved at: {}".format(this_model_path) # replace the current model new_model.save(this_model_path) - + # remove the local image path shutil.rmtree(image_data_path, ignore_errors=True) - + final_trn_acc = history.history['acc'][-1] final_val_acc = history.history['val_acc'][-1] return final_trn_acc, final_val_acc @@ -63,7 +66,7 @@ def async_retrain(model_name, # remove the local image path shutil.rmtree(image_data_path, ignore_errors=True) raise - + @michaniki_celery_app.task() def async_transfer(model_name, s3_bucket_name, @@ -76,36 +79,38 @@ def async_transfer(model_name, new_model_folder_path = os.path.join("app", "models", "InceptionV3", model_name) if not os.path.exists(new_model_folder_path): os.makedirs(new_model_folder_path) - + + print "TEMP_FOLDER->{}".format(TEMP_FOLDER) + image_data_path = API_helpers.download_a_dir_from_s3(bucket_name = s3_bucket_name, bucket_prefix = s3_bucket_prefix, local_path = TEMP_FOLDER) try: # init the transfer learning manager this_IV3_transfer = inceptionV3_transfer_retraining.InceptionTransferLeaner(model_name) - new_model, label_dict, history = this_IV3_transfer.transfer_model(image_data_path, + print "Done loading model" + print "image_data_path or local_dir:{}".format(image_data_path) + new_model, label_dict, history = this_IV3_transfer.transfer_model(image_data_path, nb_epoch = INV3_TRANSFER_NB_EPOCH, batch_size = INV3_TRANSFER_BATCH_SIZE) - + # save the model .h5 file and the class label file new_model_path = os.path.join(new_model_folder_path, model_name + ".h5") new_label_path = os.path.join(new_model_folder_path, model_name + ".json") new_model.save(new_model_path) API_helpers.save_classes_label_dict(label_dict, new_label_path) print "* Celery Transfer: New Model Saved at: {}".format(new_model_path) - + # delete the image folder here: shutil.rmtree(image_data_path, ignore_errors=True) - + # return the train and val acc: final_trn_acc = history.history['acc'][-1] final_val_acc = history.history['val_acc'][-1] return final_trn_acc, final_val_acc - + except Exception as err: # catch any error shutil.rmtree(new_model_folder_path, ignore_errors=True) shutil.rmtree(image_data_path, ignore_errors=True) raise - - \ No newline at end of file diff --git a/src/config.py b/src/config.py index 5fcf0de..dd145a5 100644 --- a/src/config.py +++ b/src/config.py @@ -11,6 +11,7 @@ INCEPTIONV3_IMAGE_QUEUE = env.str('INCEPTIONV3_IMAGE_QUEUE', default='inceptionV3_image_queue') INV3_TRANSFER_NB_EPOCH = env.str('INV3_TRANSFER_NB_EPOCH', default=3) INV3_TRANSFER_BATCH_SIZE = env.str('INV3_TRANSFER_BATCH_SIZE', default=2) +SENTIMENT_TEXT_QUEUE = env.str('SENTIMENT_TEXT_QUEUE', default = 'sentimentV1_text_queue') # setting for mysql db # parsed from environment variables @@ -22,4 +23,4 @@ # redis url for celery BROKER_URL = env.str('BROKER_URL', default='redis://redis:6379/0') -BACKEND_URL = env.str('BACKEND_URL', default='redis://redis:6379/0') \ No newline at end of file +BACKEND_URL = env.str('BACKEND_URL', default='redis://redis:6379/0') diff --git a/src/requirements.txt b/src/requirements.txt index db9ffb5..4d6b91d 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -95,3 +95,4 @@ urllib3==1.23 uWSGI==2.0.17 wcwidth==0.1.7 Werkzeug==0.14.1 +textblob==0.15.0 diff --git a/src/requirementssenti.txt b/src/requirementssenti.txt new file mode 100644 index 0000000..a889dbe --- /dev/null +++ b/src/requirementssenti.txt @@ -0,0 +1,154 @@ +absl-py==0.6.1 +argh==0.26.2 +asn1crypto==0.24.0 +astor==0.7.1 +backcall==0.1.0 +backports-abc==0.5 +base58==1.0.3 +beautifulsoup4==4.6.3 +bleach==3.0.2 +boto==2.49.0 +boto3==1.9.78 +botocore==1.12.78 +Bottleneck==1.2.1 +bz2file==0.98 +certifi==2018.11.29 +cffi==1.11.5 +chardet==3.0.4 +Click==7.0 +cryptography==2.4.2 +cycler==0.10.0 +cymem==2.0.2 +cytoolz==0.9.0.1 +dataclasses==0.6 +decorator==4.3.0 +Deprecated==1.2.4 +dill==0.2.8.2 +docutils==0.14 +entrypoints==0.2.3 +enum-compat==0.0.2 +fastai==1.0.40 +fastprogress==0.1.18 +flair==0.4.0 +Flask==1.0.2 +future==0.17.1 +future-fstrings==0.4.5 +futures==3.1.1 +gast==0.2.2 +gensim==3.4.0 +gluonnlp==0.5.0.post0 +graphviz==0.8.4 +grpcio==1.17.1 +h5py==2.9.0 +hyperopt==0.1.1 +idna==2.8 +ipykernel==5.1.0 +ipython==7.2.0 +ipython-genutils==0.2.0 +ipywidgets==7.4.2 +itsdangerous==1.1.0 +jedi==0.13.2 +Jinja2==2.10 +jmespath==0.9.3 +jsonschema==2.6.0 +jupyter==1.0.0 +jupyter-client==5.2.4 +jupyter-console==6.0.0 +jupyter-core==4.4.0 +Keras==2.2.4 +Keras-Applications==1.0.6 +Keras-Preprocessing==1.0.5 +kiwisolver==1.0.1 +Markdown==3.0.1 +MarkupSafe==1.1.0 +matplotlib==3.0.0 +mistune==0.8.4 +mkl-fft==1.0.6 +mkl-random==1.0.1 +mock==2.0.0 +mpld3==0.3 +msgpack==0.5.6 +msgpack-numpy==0.4.3.2 +murmurhash==1.0.1 +mxnet-cu90==1.3.1 +mypy==0.650 +mypy-extensions==0.4.1 +networkx==2.2 +nltk==3.4 +notebook==5.7.4 +numexpr==2.6.8 +numpy==1.14.6 +olefile==0.46 +packaging==18.0 +pandas==0.23.4 +pandocfilters==1.4.2 +parso==0.3.1 +pathtools==0.1.2 +pbr==5.1.1 +pexpect==4.6.0 +pickleshare==0.7.5 +Pillow==5.4.1 +plac==0.9.6 +preshed==2.0.1 +prometheus-client==0.5.0 +prompt-toolkit==2.0.7 +protobuf==3.6.1 +psutil==5.4.8 +ptyprocess==0.6.0 +pycparser==2.19 +Pygments==2.3.1 +pymongo==3.7.2 +pyOpenSSL==18.0.0 +pyparsing==2.3.0 +PySocks==1.6.8 +python-dateutil==2.7.5 +pytorch-pretrained-bert==0.3.0 +pytz==2018.7 +PyYAML==3.13 +pyzmq==17.1.2 +qtconsole==4.4.3 +redis==3.0.1 +regex==2018.1.10 +requests==2.21.0 +s3transfer==0.1.13 +scikit-learn==0.20.1 +scipy==1.1.0 +seaborn==0.9.0 +segtok==1.5.7 +Send2Trash==1.5.0 +singledispatch==3.4.0.3 +six==1.12.0 +sklearn==0.0 +smart-open==1.8.0 +spacy==2.0.18 +sqlitedict==1.6.0 +streamlit==0.23.0 +tensorboard==1.12.2 +tensorflow==1.12.0 +tensorflow-hub==0.2.0 +termcolor==1.1.0 +terminado==0.8.1 +testpath==0.4.2 +textblob==0.15.2 +thinc==6.12.1 +tokenize-rt==2.1.0 +toml==0.10.0 +toolz==0.9.0 +torch==1.0.0 +torchtext==0.3.1 +torchvision==0.2.1 +tornado==5.1.1 +tqdm==4.26.0 +traitlets==4.3.2 +typed-ast==1.1.1 +typing==3.6.4 +tzlocal==1.5.1 +ujson==1.35 +urllib3==1.24.1 +watchdog==0.9.0 +wcwidth==0.1.7 +webencodings==0.5.1 +Werkzeug==0.14.1 +widgetsnbextension==3.4.2 +wrapt==1.10.11 +xgboost==0.81