Skip to content

Removed other references to boto3 client #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 24, 2025
Merged

Removed other references to boto3 client #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
129 changes: 66 additions & 63 deletions chatfuncs/aws_functions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,13 @@
import tempfile
import os
from chatfuncs.helper_functions import get_or_create_env_var
from chatfuncs.config import AWS_REGION, RUN_AWS_FUNCTIONS, QA_CHATBOT_BUCKET

PandasDataFrame = Type[pd.DataFrame]

# Get AWS credentials if required
bucket_name=""
bucket_name=QA_CHATBOT_BUCKET

RUN_AWS_FUNCTIONS = get_or_create_env_var("RUN_AWS_FUNCTIONS", "0")
print(f'The value of RUN_AWS_FUNCTIONS is {RUN_AWS_FUNCTIONS}')

AWS_REGION = get_or_create_env_var('AWS_REGION', '')
print(f'The value of AWS_REGION is {AWS_REGION}')

if RUN_AWS_FUNCTIONS == "1":
try:
Expand Down Expand Up @@ -49,67 +45,71 @@ def get_assumed_role_info():
# Download direct from S3 - requires login credentials
def download_file_from_s3(bucket_name, key, local_file_path):

s3 = boto3.client('s3')
s3.download_file(bucket_name, key, local_file_path)
print(f"File downloaded from S3: s3://{bucket_name}/{key} to {local_file_path}")
if RUN_AWS_FUNCTIONS == "1":

s3 = boto3.client('s3')
s3.download_file(bucket_name, key, local_file_path)
print(f"File downloaded from S3: s3://{bucket_name}/{key} to {local_file_path}")

def download_folder_from_s3(bucket_name, s3_folder, local_folder):
"""
Download all files from an S3 folder to a local folder.
"""
s3 = boto3.client('s3')
if RUN_AWS_FUNCTIONS == "1":
s3 = boto3.client('s3')

# List objects in the specified S3 folder
response = s3.list_objects_v2(Bucket=bucket_name, Prefix=s3_folder)
# List objects in the specified S3 folder
response = s3.list_objects_v2(Bucket=bucket_name, Prefix=s3_folder)

# Download each object
for obj in response.get('Contents', []):
# Extract object key and construct local file path
object_key = obj['Key']
local_file_path = os.path.join(local_folder, os.path.relpath(object_key, s3_folder))
# Download each object
for obj in response.get('Contents', []):
# Extract object key and construct local file path
object_key = obj['Key']
local_file_path = os.path.join(local_folder, os.path.relpath(object_key, s3_folder))

# Create directories if necessary
os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
# Create directories if necessary
os.makedirs(os.path.dirname(local_file_path), exist_ok=True)

# Download the object
try:
s3.download_file(bucket_name, object_key, local_file_path)
print(f"Downloaded 's3://{bucket_name}/{object_key}' to '{local_file_path}'")
except Exception as e:
print(f"Error downloading 's3://{bucket_name}/{object_key}':", e)
# Download the object
try:
s3.download_file(bucket_name, object_key, local_file_path)
print(f"Downloaded 's3://{bucket_name}/{object_key}' to '{local_file_path}'")
except Exception as e:
print(f"Error downloading 's3://{bucket_name}/{object_key}':", e)

def download_files_from_s3(bucket_name, s3_folder, local_folder, filenames):
"""
Download specific files from an S3 folder to a local folder.
"""
s3 = boto3.client('s3')
if RUN_AWS_FUNCTIONS == "1":
s3 = boto3.client('s3')

print("Trying to download file: ", filenames)
print("Trying to download file: ", filenames)

if filenames == '*':
# List all objects in the S3 folder
print("Trying to download all files in AWS folder: ", s3_folder)
response = s3.list_objects_v2(Bucket=bucket_name, Prefix=s3_folder)
if filenames == '*':
# List all objects in the S3 folder
print("Trying to download all files in AWS folder: ", s3_folder)
response = s3.list_objects_v2(Bucket=bucket_name, Prefix=s3_folder)

print("Found files in AWS folder: ", response.get('Contents', []))
print("Found files in AWS folder: ", response.get('Contents', []))

filenames = [obj['Key'].split('/')[-1] for obj in response.get('Contents', [])]
filenames = [obj['Key'].split('/')[-1] for obj in response.get('Contents', [])]

print("Found filenames in AWS folder: ", filenames)
print("Found filenames in AWS folder: ", filenames)

for filename in filenames:
object_key = os.path.join(s3_folder, filename)
local_file_path = os.path.join(local_folder, filename)
for filename in filenames:
object_key = os.path.join(s3_folder, filename)
local_file_path = os.path.join(local_folder, filename)

# Create directories if necessary
os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
# Create directories if necessary
os.makedirs(os.path.dirname(local_file_path), exist_ok=True)

# Download the object
try:
s3.download_file(bucket_name, object_key, local_file_path)
print(f"Downloaded 's3://{bucket_name}/{object_key}' to '{local_file_path}'")
except Exception as e:
print(f"Error downloading 's3://{bucket_name}/{object_key}':", e)
# Download the object
try:
s3.download_file(bucket_name, object_key, local_file_path)
print(f"Downloaded 's3://{bucket_name}/{object_key}' to '{local_file_path}'")
except Exception as e:
print(f"Error downloading 's3://{bucket_name}/{object_key}':", e)

def upload_file_to_s3(local_file_paths:List[str], s3_key:str, s3_bucket:str=bucket_name):
"""
Expand All @@ -124,30 +124,33 @@ def upload_file_to_s3(local_file_paths:List[str], s3_key:str, s3_bucket:str=buck
- Message as variable/printed to console
"""
final_out_message = []
final_out_message_str = ""

s3_client = boto3.client('s3')
if RUN_AWS_FUNCTIONS == "1":

if isinstance(local_file_paths, str):
local_file_paths = [local_file_paths]
s3_client = boto3.client('s3')

for file in local_file_paths:
try:
# Get file name off file path
file_name = os.path.basename(file)
if isinstance(local_file_paths, str):
local_file_paths = [local_file_paths]

s3_key_full = s3_key + file_name
print("S3 key: ", s3_key_full)
for file in local_file_paths:
try:
# Get file name off file path
file_name = os.path.basename(file)

s3_client.upload_file(file, s3_bucket, s3_key_full)
out_message = "File " + file_name + " uploaded successfully!"
print(out_message)

except Exception as e:
out_message = f"Error uploading file(s): {e}"
print(out_message)
s3_key_full = s3_key + file_name
print("S3 key: ", s3_key_full)

s3_client.upload_file(file, s3_bucket, s3_key_full)
out_message = "File " + file_name + " uploaded successfully!"
print(out_message)

except Exception as e:
out_message = f"Error uploading file(s): {e}"
print(out_message)

final_out_message.append(out_message)
final_out_message_str = '\n'.join(final_out_message)
final_out_message.append(out_message)
final_out_message_str = '\n'.join(final_out_message)

return final_out_message_str

Expand Down
2 changes: 1 addition & 1 deletion chatfuncs/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ def add_folder_to_path(folder_path: str):
AWS_SECRET_KEY = get_or_create_env_var('AWS_SECRET_KEY', '')
if AWS_SECRET_KEY: print(f'AWS_SECRET_KEY found in environment variables')

DOCUMENT_REDACTION_BUCKET = get_or_create_env_var('DOCUMENT_REDACTION_BUCKET', '')
QA_CHATBOT_BUCKET = get_or_create_env_var('QA_CHATBOT_BUCKET', '')

# Custom headers e.g. if routing traffic through Cloudfront
# Retrieving or setting CUSTOM_HEADER
Expand Down
Loading