Skip to content

Commit 60425fc

Browse files
seanpedrick-caseseanpedrick-case
authored
Merge pull request #3 from seanpedrick-case/dev
Removed other references to boto3 client
2 parents aa5c77e + b4066c5 commit 60425fc

File tree

2 files changed

+67
-64
lines changed

2 files changed

+67
-64
lines changed

‎chatfuncs/aws_functions.py‎

Lines changed: 66 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,13 @@
44
import tempfile
55
import os
66
from chatfuncs.helper_functions import get_or_create_env_var
7+
from chatfuncs.config import AWS_REGION, RUN_AWS_FUNCTIONS, QA_CHATBOT_BUCKET
78

89
PandasDataFrame = Type[pd.DataFrame]
910

1011
# Get AWS credentials if required
11-
bucket_name=""
12+
bucket_name=QA_CHATBOT_BUCKET
1213

13-
RUN_AWS_FUNCTIONS = get_or_create_env_var("RUN_AWS_FUNCTIONS", "0")
14-
print(f'The value of RUN_AWS_FUNCTIONS is {RUN_AWS_FUNCTIONS}')
15-
16-
AWS_REGION = get_or_create_env_var('AWS_REGION', '')
17-
print(f'The value of AWS_REGION is {AWS_REGION}')
1814

1915
if RUN_AWS_FUNCTIONS == "1":
2016
try:
@@ -49,67 +45,71 @@ def get_assumed_role_info():
4945
# Download direct from S3 - requires login credentials
5046
def download_file_from_s3(bucket_name, key, local_file_path):
5147

52-
s3 = boto3.client('s3')
53-
s3.download_file(bucket_name, key, local_file_path)
54-
print(f"File downloaded from S3: s3://{bucket_name}/{key} to {local_file_path}")
48+
if RUN_AWS_FUNCTIONS == "1":
49+
50+
s3 = boto3.client('s3')
51+
s3.download_file(bucket_name, key, local_file_path)
52+
print(f"File downloaded from S3: s3://{bucket_name}/{key} to {local_file_path}")
5553

5654
def download_folder_from_s3(bucket_name, s3_folder, local_folder):
5755
"""
5856
Download all files from an S3 folder to a local folder.
5957
"""
60-
s3 = boto3.client('s3')
58+
if RUN_AWS_FUNCTIONS == "1":
59+
s3 = boto3.client('s3')
6160

62-
# List objects in the specified S3 folder
63-
response = s3.list_objects_v2(Bucket=bucket_name, Prefix=s3_folder)
61+
# List objects in the specified S3 folder
62+
response = s3.list_objects_v2(Bucket=bucket_name, Prefix=s3_folder)
6463

65-
# Download each object
66-
for obj in response.get('Contents', []):
67-
# Extract object key and construct local file path
68-
object_key = obj['Key']
69-
local_file_path = os.path.join(local_folder, os.path.relpath(object_key, s3_folder))
64+
# Download each object
65+
for obj in response.get('Contents', []):
66+
# Extract object key and construct local file path
67+
object_key = obj['Key']
68+
local_file_path = os.path.join(local_folder, os.path.relpath(object_key, s3_folder))
7069

71-
# Create directories if necessary
72-
os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
70+
# Create directories if necessary
71+
os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
7372

74-
# Download the object
75-
try:
76-
s3.download_file(bucket_name, object_key, local_file_path)
77-
print(f"Downloaded 's3://{bucket_name}/{object_key}' to '{local_file_path}'")
78-
except Exception as e:
79-
print(f"Error downloading 's3://{bucket_name}/{object_key}':", e)
73+
# Download the object
74+
try:
75+
s3.download_file(bucket_name, object_key, local_file_path)
76+
print(f"Downloaded 's3://{bucket_name}/{object_key}' to '{local_file_path}'")
77+
except Exception as e:
78+
print(f"Error downloading 's3://{bucket_name}/{object_key}':", e)
8079

8180
def download_files_from_s3(bucket_name, s3_folder, local_folder, filenames):
8281
"""
8382
Download specific files from an S3 folder to a local folder.
8483
"""
85-
s3 = boto3.client('s3')
84+
if RUN_AWS_FUNCTIONS == "1":
85+
s3 = boto3.client('s3')
8686

87-
print("Trying to download file: ", filenames)
87+
print("Trying to download file: ", filenames)
8888

89-
if filenames == '*':
90-
# List all objects in the S3 folder
91-
print("Trying to download all files in AWS folder: ", s3_folder)
92-
response = s3.list_objects_v2(Bucket=bucket_name, Prefix=s3_folder)
89+
if filenames == '*':
90+
# List all objects in the S3 folder
91+
print("Trying to download all files in AWS folder: ", s3_folder)
92+
response = s3.list_objects_v2(Bucket=bucket_name, Prefix=s3_folder)
9393

94-
print("Found files in AWS folder: ", response.get('Contents', []))
94+
print("Found files in AWS folder: ", response.get('Contents', []))
9595

96-
filenames = [obj['Key'].split('/')[-1] for obj in response.get('Contents', [])]
96+
filenames = [obj['Key'].split('/')[-1] for obj in response.get('Contents', [])]
9797

98-
print("Found filenames in AWS folder: ", filenames)
98+
print("Found filenames in AWS folder: ", filenames)
9999

100-
for filename in filenames:
101-
object_key = os.path.join(s3_folder, filename)
102-
local_file_path = os.path.join(local_folder, filename)
100+
for filename in filenames:
101+
object_key = os.path.join(s3_folder, filename)
102+
local_file_path = os.path.join(local_folder, filename)
103103

104-
# Create directories if necessary
105-
os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
104+
# Create directories if necessary
105+
os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
106106

107-
# Download the object
108-
try:
109-
s3.download_file(bucket_name, object_key, local_file_path)
110-
print(f"Downloaded 's3://{bucket_name}/{object_key}' to '{local_file_path}'")
111-
except Exception as e:
112-
print(f"Error downloading 's3://{bucket_name}/{object_key}':", e)
107+
# Download the object
108+
try:
109+
s3.download_file(bucket_name, object_key, local_file_path)
110+
print(f"Downloaded 's3://{bucket_name}/{object_key}' to '{local_file_path}'")
111+
except Exception as e:
112+
print(f"Error downloading 's3://{bucket_name}/{object_key}':", e)
113113

114114
def upload_file_to_s3(local_file_paths:List[str], s3_key:str, s3_bucket:str=bucket_name):
115115
"""
@@ -124,30 +124,33 @@ def upload_file_to_s3(local_file_paths:List[str], s3_key:str, s3_bucket:str=buck
124124
- Message as variable/printed to console
125125
"""
126126
final_out_message = []
127+
final_out_message_str = ""
127128

128-
s3_client = boto3.client('s3')
129+
if RUN_AWS_FUNCTIONS == "1":
129130

130-
if isinstance(local_file_paths, str):
131-
local_file_paths = [local_file_paths]
131+
s3_client = boto3.client('s3')
132132

133-
for file in local_file_paths:
134-
try:
135-
# Get file name off file path
136-
file_name = os.path.basename(file)
133+
if isinstance(local_file_paths, str):
134+
local_file_paths = [local_file_paths]
137135

138-
s3_key_full = s3_key + file_name
139-
print("S3 key: ", s3_key_full)
136+
for file in local_file_paths:
137+
try:
138+
# Get file name off file path
139+
file_name = os.path.basename(file)
140140

141-
s3_client.upload_file(file, s3_bucket, s3_key_full)
142-
out_message = "File " + file_name + " uploaded successfully!"
143-
print(out_message)
144-
145-
except Exception as e:
146-
out_message = f"Error uploading file(s): {e}"
147-
print(out_message)
141+
s3_key_full = s3_key + file_name
142+
print("S3 key: ", s3_key_full)
143+
144+
s3_client.upload_file(file, s3_bucket, s3_key_full)
145+
out_message = "File " + file_name + " uploaded successfully!"
146+
print(out_message)
147+
148+
except Exception as e:
149+
out_message = f"Error uploading file(s): {e}"
150+
print(out_message)
148151

149-
final_out_message.append(out_message)
150-
final_out_message_str = '\n'.join(final_out_message)
152+
final_out_message.append(out_message)
153+
final_out_message_str = '\n'.join(final_out_message)
151154

152155
return final_out_message_str
153156

‎chatfuncs/config.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ def add_folder_to_path(folder_path: str):
107107
AWS_SECRET_KEY = get_or_create_env_var('AWS_SECRET_KEY', '')
108108
if AWS_SECRET_KEY: print(f'AWS_SECRET_KEY found in environment variables')
109109

110-
DOCUMENT_REDACTION_BUCKET = get_or_create_env_var('DOCUMENT_REDACTION_BUCKET', '')
110+
QA_CHATBOT_BUCKET = get_or_create_env_var('QA_CHATBOT_BUCKET', '')
111111

112112
# Custom headers e.g. if routing traffic through Cloudfront
113113
# Retrieving or setting CUSTOM_HEADER

0 commit comments

Comments
 (0)