WebCrawlingApplicationTesting/sklearn_transformations.py at master · jlug331221/WebCrawlingApplicationTesting · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
import os, re
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.decomposition import TruncatedSVD

CURRENT_DIR = os.path.dirname(__file__)

'''
# Return bag of words transformation on feature_vectors (for each document).
#
# Output is written to 'sklearn_transformation_output/bag_of_words.txt'.
#
# bag of words := count of significant words in each feature vector.
# Each bag of words transformation produces a document-term matrix
# rows    := length of the list being transformed
# columns := unique words in each feature vector
#
# Example:
#
  ['first name', 'id', 'firstname', 'name', 'firstname', 'maxlength', '45', 'type', 'text']
  [  [0 1 0 0 0 1 0 0]
     [0 0 0 1 0 0 0 0]
     [0 0 1 0 0 0 0 0]
     [0 0 0 0 0 1 0 0]
     [0 0 1 0 0 0 0 0]
     [0 0 0 0 1 0 0 0]
     [1 0 0 0 0 0 0 0]
     [0 0 0 0 0 0 0 1]
     [0 0 0 0 0 0 1 0] ]
  Feature names: ['45', 'first', 'firstname', 'id', 'maxlength', 'name', 'text', 'type']
#
# 9 x 8 document-term matrix
#
# The first sub-list := [0 1 0 0 0 1 0 0] says that 'first' and 'name' appear one time in the
# document.
# This is verified by printing the feature names of the vectorizer, which is the last list in the
# example.
'''
def bag_of_words_transformation(feature_vectors):
  bag_of_words = dict()
  count_vectorizer = CountVectorizer()

  with open(CURRENT_DIR + '/sklearn_transformation_output/bag_of_words.txt', 'w') as BoW_output:
    for key in feature_vectors.keys():
      bag_of_words[key] = []
      BoW_output.write(key + '\n')

      feature_vector_words = []
      vector_word_string = ''
      for feature_vector in feature_vectors.get(key):
        BoW_output.write(str(feature_vector) + '\n')

        for feature_vector_word in feature_vector:
          vector_word_string += str(feature_vector_word) + ' '

        feature_vector_words.append(vector_word_string)

        vector_word_string = ''

      # remove 'id' and 'maxlength' from feature_vector_words
      feature_vector_words_without_id_or_maxlength = []
      for vector_words in feature_vector_words:
        vector_words = vector_words.replace('id', '')
        vector_words = vector_words.replace('maxlength', '')
        feature_vector_words_without_id_or_maxlength.append(vector_words)

      bag_of_words[key] = count_vectorizer\
        .fit_transform(feature_vector_words_without_id_or_maxlength)

      BoW_output.write(str(bag_of_words[key]) + '\n')
      # BoW_output.write(str(bag_of_words[key].toarray()) + '\n')
      BoW_output.write(str(bag_of_words[key].shape) + '\n')
      BoW_output.write('Feature names: ' + str(count_vectorizer.get_feature_names()) + '\n\n')
      BoW_output.write('*************************************************************' + '\n\n')

  return bag_of_words

'''
# Apply and return the TF-IDF transformation on bag_of_words. The transformation converts
# the counts in bag_of_words to real-value weights (real numbers). TF-IDF measures the
# relevance of the word and not the frequency.
#
# TF-IDF first measures the number of times a word appears in a document. The inverse
# document frequency aspect handles words such as 'and' or 'but' which appear in all
# documents and those words are given less relevance (weight). The result of TF-IDF is
# words that are frequent and distinctive.
'''
def tfidf_transformation(bag_of_words):
  tfidf = dict()
  tfidf_transformer = TfidfTransformer()

  with open(CURRENT_DIR + '/sklearn_transformation_output/tfidf.txt', 'w') as tfidf_output:
    for key in bag_of_words.keys():
      tfidf[key] = []
      tfidf_output.write(key + '\n')

      tfidf[key] = tfidf_transformer.fit_transform(bag_of_words[key])

      tfidf_output.write(str(tfidf[key]) + '\n\n')

    tfidf_output.write('*************************************************************' + '\n\n')

  return tfidf

'''
# Apply and return the latent semantic analysis (LSA) transformation on tfidf. This
# transformation reduces the dimension of the vector space by means of singular value
# decomposition (SVD).
'''
def LSA_transformation(tfidf):
  LSA = dict()
  svd = TruncatedSVD()

  with open(CURRENT_DIR + '/sklearn_transformation_output/LSA.txt', 'w') as LSA_output:
    for key in tfidf.keys():
      LSA[key] = []
      LSA_output.write(key + '\n')

      LSA[key] = svd.fit_transform(tfidf[key])

      LSA_output.write(str(list(enumerate(LSA[key]))) + '\n\n')
      LSA_output.write('V^T: ' + str(svd.components_) + '\n\n')

      LSA_output.write('*************************************************************' + '\n\n')

  return LSA