main.py.py

import streamlit as st
import pandas as pd
import tweepy
from textblob import TextBlob
from wordcloud import WordCloud
from collections import Counter
import matplotlib.pyplot as plt
import plotly.express as px
import re
from nltk.corpus import stopwords
import nltk
import pyLDAvis.gensim as gensim_vis
import pyLDAvis
import gensim
from gensim import corpora

# Download necessary resources
nltk.download("stopwords")
stop_words = set(stopwords.words("english"))

# Helper Functions
def fetch_live_tweets(bearer_token, query, max_tweets):
    try:
        client = tweepy.Client(bearer_token=bearer_token)
        tweets = client.search_recent_tweets(
            query=query, 
            max_results=max_tweets, 
            tweet_fields=["text", "created_at", "lang"]
        )
        tweet_data = [
            {
                "tweet": tweet.text,
                "created_at": tweet.created_at,
                "lang": tweet.lang
            }
            for tweet in tweets.data
        ]
        return pd.DataFrame(tweet_data)
    except Exception as e:
        st.error(f"Error fetching tweets: {e}")
        return pd.DataFrame()

def preprocess_text(text):
    text = re.sub(r"http\S+", "", text)  # Remove links
    text = re.sub(r"[^a-zA-Z\s]", "", text)  # Remove special characters
    text = text.lower()  # Convert to lowercase
    words = [word for word in text.split() if word not in stop_words]
    return " ".join(words)

def analyze_sentiment(tweet):
    analysis = TextBlob(tweet)
    if analysis.sentiment.polarity > 0:
        return "Positive"
    elif analysis.sentiment.polarity < 0:
        return "Negative"
    else:
        return "Neutral"

def extract_keywords(tweets):
    all_words = " ".join(tweets).split()
    filtered_words = [word.lower() for word in all_words if word not in stop_words and len(word) > 2]
    return Counter(filtered_words).most_common(10)

def generate_wordcloud(tweets):
    all_words = " ".join(tweets)
    wordcloud = WordCloud(width=800, height=400, background_color="white").generate(all_words)
    return wordcloud

def perform_lda_analysis(tweets, n_topics=5):
    tweets_cleaned = tweets.apply(preprocess_text)
    tokenized_tweets = [tweet.split() for tweet in tweets_cleaned]
    dictionary = corpora.Dictionary(tokenized_tweets)
    corpus = [dictionary.doc2bow(tweet) for tweet in tokenized_tweets]
    lda_model = gensim.models.LdaModel(corpus, num_topics=n_topics, id2word=dictionary, passes=10)
    lda_vis = gensim_vis.prepare(lda_model, corpus, dictionary)
    return lda_vis

# Streamlit App Configuration
st.set_page_config(
    page_title="Twitter EDA and Analysis",
    layout="wide",
    initial_sidebar_state="expanded",
)

# Sidebar Configuration
st.sidebar.header("Twitter Configuration")
bearer_token = st.sidebar.text_input("Bearer Token", type="password")
search_query = st.sidebar.text_input("Search Query", value="Python Programming")
max_tweets = st.sidebar.slider("Number of Tweets to Fetch", 10, 100, 50)
n_topics = st.sidebar.slider("Number of LDA Topics", 2, 10, 5)

# Main App
st.title("Twitter EDA and Analysis")
st.write("""
This tool provides comprehensive analysis of tweets, including sentiment, keywords, trends, word clouds, and topic modeling.
""")

if st.sidebar.button("Fetch Tweets"):
    if bearer_token:
        with st.spinner("Fetching tweets..."):
            tweets_data = fetch_live_tweets(bearer_token, search_query, max_tweets)
            if not tweets_data.empty:
                st.success(f"Fetched {len(tweets_data)} tweets!")
                tweets_data["cleaned_tweet"] = tweets_data["tweet"].apply(preprocess_text)
                tweets_data["sentiment"] = tweets_data["cleaned_tweet"].apply(analyze_sentiment)
                
                # Display Raw Data
                st.header("Fetched Tweets")
                st.dataframe(tweets_data)

                # Sentiment Analysis
                st.header("Sentiment Analysis")
                sentiment_counts = tweets_data["sentiment"].value_counts()
                sentiment_df = pd.DataFrame({"Sentiment": sentiment_counts.index, "Count": sentiment_counts.values})
                fig_sentiment = px.pie(sentiment_df, values="Count", names="Sentiment", title="Sentiment Distribution")
                st.plotly_chart(fig_sentiment)

                # Keyword Analysis
                st.header("Keyword Analysis")
                keyword_freq = extract_keywords(tweets_data["cleaned_tweet"])
                keyword_df = pd.DataFrame(keyword_freq, columns=["Keyword", "Frequency"])
                st.bar_chart(keyword_df.set_index("Keyword"))

                # Word Cloud
                st.header("Word Cloud")
                wordcloud = generate_wordcloud(tweets_data["cleaned_tweet"])
                plt.figure(figsize=(10, 5))
                plt.imshow(wordcloud, interpolation="bilinear")
                plt.axis("off")
                st.pyplot(plt)

                # Time-based Trend Analysis
                st.header("Time-based Trend Analysis")
                tweets_data["created_at"] = pd.to_datetime(tweets_data["created_at"])
                tweets_data["hour"] = tweets_data["created_at"].dt.hour
                time_counts = tweets_data.groupby("hour").size().reset_index(name="Counts")
                fig_time = px.line(time_counts, x="hour", y="Counts", title="Tweet Activity Over Time")
                st.plotly_chart(fig_time)

                # Hashtag Analysis
                st.header("Hashtag Analysis")
                hashtags = tweets_data["tweet"].apply(lambda x: re.findall(r"#(\w+)", x))
                hashtags_flat = [tag.lower() for tags in hashtags for tag in tags]
                hashtag_counts = Counter(hashtags_flat).most_common(10)
                hashtag_df = pd.DataFrame(hashtag_counts, columns=["Hashtag", "Frequency"])
                st.bar_chart(hashtag_df.set_index("Hashtag"))

                # Language Distribution
                st.header("Language Distribution")
                lang_counts = tweets_data["lang"].value_counts()
                lang_df = pd.DataFrame({"Language": lang_counts.index, "Count": lang_counts.values})
                fig_lang = px.bar(lang_df, x="Language", y="Count", title="Language Distribution of Tweets")
                st.plotly_chart(fig_lang)

                # LDA Topic Modeling
                st.header("LDA Topic Modeling")
                lda_vis = perform_lda_analysis(tweets_data["cleaned_tweet"], n_topics)
                pyLDAvis.save_html(lda_vis, "lda_visualization.html")
                st.components.v1.html(open("lda_visualization.html", "r").read(), height=800)
            else:
                st.warning("No tweets found. Please check your query or API credentials.")
    else:
        st.warning("Please provide a valid Bearer Token.")