congress-legislators/Rakefile at master · helloworlddata/congress-legislators · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
require 'pathname'
DATA_DIR = Pathname 'catalog'
WRANGLE_DIR = Pathname 'wrangle'
CORRAL_DIR = WRANGLE_DIR.join('corral')
SCRIPTS = WRANGLE_DIR.join('scripts')
DIRS = {
    'fetched' => CORRAL_DIR / 'fetched',
    'compiled' => CORRAL_DIR / 'compiled',
    'published' => DATA_DIR,
}

F_FILES = {
  'sessions' => DIRS['fetched'] / 'sessions.tsv',
  'legislators-current' => DIRS['fetched'] / 'legislators-current.yaml',
  'legislators-historical' => DIRS['fetched'] / 'legislators-historical.yaml',
}


C_FILES = {
  'legislators' => DIRS['compiled'] / 'legislators.csv',
  'congresses' => DIRS['compiled'] / 'congresses.csv',
  'terms' => DIRS['compiled'] / 'terms.csv',
}


P_FILES = {
  'legislators' => DIRS['published'] / 'congress-legislators.csv',
  'congresses' => DIRS['published'] / 'congresses.csv',
  'terms' => DIRS['published'] / 'congress-terms.csv',
}


desc 'Setup the directories'
task :setup do
    DIRS.each_value do |p|
        unless p.exist?
            p.mkpath()
            puts "Created directory: #{p}"
        end
    end
end

desc "Fetch everything"
task :fetch  => [:setup] do
  F_FILES.each_value{|fn| Rake::Task[fn].execute() }
end

desc "Compile everything"
task :compile  => [:setup] do
  C_FILES.each_value{|fn| Rake::Task[fn].execute() }
end

desc "Publish everything"
task :publish  => C_FILES.values() do
  C_FILES.each_key do |key|
    sh "cp #{C_FILES[key]} #{P_FILES[key]}"
  end
end


namespace :files do
  namespace :published do
      SOURCE_URLPATH = 'https://raw.githubusercontent.com/unitedstates/congress-legislators/master'


      file C_FILES['legislators'] => [F_FILES['legislators-current'],
                                      F_FILES['legislators-historical']] do

        sh %Q{
python #{SCRIPTS / 'collate_legislators.py'} \
  #{F_FILES['legislators-current']} \
  #{F_FILES['legislators-historical']} \
> #{C_FILES['legislators']}
        }
      end


      desc "The file containing congress and numbers"
      file C_FILES['congresses'] => F_FILES['sessions'] do
        sh %Q{
python #{SCRIPTS / 'collate_congresses.py'} \
  #{F_FILES['sessions']} \
  > #{C_FILES['congresses']}
          }
      end


      file C_FILES['terms'] => [F_FILES['legislators-current'],
                                      F_FILES['legislators-historical']] do

        sh %Q{
python #{SCRIPTS / 'collate_terms.py'} \
  #{F_FILES['legislators-current']} \
  #{F_FILES['legislators-historical']} \
> #{C_FILES['terms']}
        }
      end
  end


  namespace :fetched do
      file F_FILES['sessions'] do
        sh %Q{
              curl https://www.govtrack.us/data/us/sessions.tsv \
              -o #{F_FILES['sessions']}
            }
      end

      file F_FILES['legislators-current'] do
        sh %Q{
              curl #{SOURCE_URLPATH}/legislators-current.yaml \
              -o #{F_FILES['legislators-current']}
            }
      end

      file F_FILES['legislators-historical'] do
        sh %Q{
              curl #{SOURCE_URLPATH}/legislators-historical.yaml \
              -o #{F_FILES['legislators-historical']}
            }
      end
  end
end