Skip to content

lab-dw-data-structuring-and-combining-lab-7 #673

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
michaelgrasshopper wants to merge 2 commits into
base: main
Choose a base branch
from
Open

lab-dw-data-structuring-and-combining-lab-7 #673

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cd6ae5e
lab-dw-data-structuring-and-combining-lab-7
michaelgrasshopper Jun 4, 2026
2f843d4
lab-dw-data-structuring-and-combining-lab-7
michaelgrasshopper Jun 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
205 changes: 202 additions & 3 deletions lab-dw-data-structuring-and-combining.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,166 @@
},
"outputs": [],
"source": [
"# Your code goes here"
"import pandas as pd\n",
" \n",
"# ─────────────────────────────────────────────────────────────\n",
"# 1. LOAD RAW DATA\n",
"# ─────────────────────────────────────────────────────────────\n",
"URL = \"https://raw.githubusercontent.com/data-bootcamp-v4/data/main/marketing_customer_analysis_clean.csv\"\n",
"df = pd.read_csv(URL)\n",
" \n",
"print(\"=\" * 60)\n",
"print(f\"RAW shape: {df.shape}\")\n",
"print(\"=\" * 60)\n",
" \n",
" \n",
"# ─────────────────────────────────────────────────────────────\n",
"# 2. COLUMN NAMES\n",
"# - Drop the unnamed index column (artefact from previous save)\n",
"# - Fix typo: 'employmentstatus' → 'employment_status'\n",
"# - Rename 'month' → 'month_number' for clarity\n",
"# ─────────────────────────────────────────────────────────────\n",
"df.drop(columns=[\"unnamed:_0\"], inplace=True)\n",
" \n",
"df.rename(columns={\n",
" \"employmentstatus\": \"employment_status\",\n",
" \"month\": \"month_number\",\n",
"}, inplace=True)\n",
" \n",
"print(\"\\nColumns after rename:\")\n",
"print(df.columns.tolist())\n",
" \n",
" \n",
"# ─────────────────────────────────────────────────────────────\n",
"# 3. DATA TYPES\n",
"# - effective_to_date → datetime\n",
"# - number_of_open_complaints → int (was float 0.0–5.0\n",
"# due to earlier fraction-parsing step)\n",
"# ─────────────────────────────────────────────────────────────\n",
"df[\"effective_to_date\"] = pd.to_datetime(df[\"effective_to_date\"], format=\"%Y-%m-%d\")\n",
"df[\"number_of_open_complaints\"] = df[\"number_of_open_complaints\"].astype(int)\n",
" \n",
"print(\"\\nDtypes after casting:\")\n",
"print(df.dtypes)\n",
" \n",
" \n",
"# ─────────────────────────────────────────────────────────────\n",
"# 4. DUPLICATES\n",
"# Customers legitimately appear multiple times (multiple\n",
"# policies), so we only drop *fully* duplicate rows.\n",
"# ─────────────────────────────────────────────────────────────\n",
"dupes = df.duplicated().sum()\n",
"print(f\"\\nFully duplicate rows removed: {dupes}\")\n",
"df.drop_duplicates(inplace=True)\n",
"df.reset_index(drop=True, inplace=True)\n",
" \n",
" \n",
"# ─────────────────────────────────────────────────────────────\n",
"# 5. NULL VALUES\n",
"# No nulls present in this dataset, but we apply a robust\n",
"# strategy so the function stays reusable on future data:\n",
"# - Numeric columns → fill with column median\n",
"# - String columns → fill with column mode\n",
"# ─────────────────────────────────────────────────────────────\n",
"print(\"\\nNull counts per column:\")\n",
"print(df.isnull().sum())\n",
" \n",
"num_cols = df.select_dtypes(include=\"number\").columns\n",
"str_cols = df.select_dtypes(include=\"str\").columns\n",
" \n",
"for col in num_cols:\n",
" df[col] = df[col].fillna(df[col].median())\n",
" \n",
"for col in str_cols:\n",
" df[col] = df[col].fillna(df[col].mode()[0])\n",
" \n",
" \n",
"# ─────────────────────────────────────────────────────────────\n",
"# 6. CATEGORICAL CONSISTENCY\n",
"# - Strip whitespace from all string columns\n",
"# - Expand gender codes for readability: M → Male, F → Female\n",
"# - Note: vehicle_type only has one unique value ('A') —\n",
"# flagged as a low-variance column (no action needed)\n",
"# ─────────────────────────────────────────────────────────────\n",
"for col in str_cols:\n",
" df[col] = df[col].str.strip()\n",
" \n",
"df[\"gender\"] = df[\"gender\"].map({\"M\": \"Male\", \"F\": \"Female\"})\n",
" \n",
"print(\"\\nUnique vehicle_type values (low-variance):\", df[\"vehicle_type\"].unique().tolist())\n",
" \n",
" \n",
"# ─────────────────────────────────────────────────────────────\n",
"# 7. DERIVED COLUMN\n",
"# Add month_name for human-readable reporting\n",
"# ─────────────────────────────────────────────────────────────\n",
"month_map = {\n",
" 1: \"January\", 2: \"February\", 3: \"March\",\n",
" 4: \"April\", 5: \"May\", 6: \"June\",\n",
" 7: \"July\", 8: \"August\", 9: \"September\",\n",
" 10: \"October\", 11: \"November\", 12: \"December\",\n",
"}\n",
"df[\"month_name\"] = df[\"month_number\"].map(month_map)\n",
" \n",
" \n",
"# ─────────────────────────────────────────────────────────────\n",
"# 8. COLUMN ORDER (logical grouping)\n",
"# ─────────────────────────────────────────────────────────────\n",
"df = df[[\n",
" \"customer\",\n",
" # demographics\n",
" \"state\", \"gender\", \"education\", \"marital_status\",\n",
" \"employment_status\", \"income\", \"location_code\",\n",
" # policy\n",
" \"policy_type\", \"policy\", \"coverage\",\n",
" \"renew_offer_type\", \"sales_channel\",\n",
" \"effective_to_date\", \"month_number\", \"month_name\",\n",
" # financial\n",
" \"customer_lifetime_value\", \"monthly_premium_auto\",\n",
" \"total_claim_amount\",\n",
" # vehicle\n",
" \"vehicle_class\", \"vehicle_size\", \"vehicle_type\",\n",
" # behaviour\n",
" \"response\", \"number_of_open_complaints\",\n",
" \"number_of_policies\", \"months_since_last_claim\",\n",
" \"months_since_policy_inception\",\n",
"]]\n",
" \n",
"print(f\"\\nCLEAN shape: {df.shape}\")\n",
"print(\"\\nFirst 3 rows:\")\n",
"print(df.head(3).to_string())\n",
" \n",
" \n",
"# ─────────────────────────────────────────────────────────────\n",
"# 9. PIVOT → LONG FORMAT\n",
"# Complaints by policy type & month\n",
"# ─────────────────────────────────────────────────────────────\n",
"pivot = df.pivot_table(\n",
" values=\"number_of_open_complaints\",\n",
" index=\"policy_type\",\n",
" columns=\"month_name\",\n",
" aggfunc=\"sum\",\n",
").round(2)\n",
" \n",
"long_df = (\n",
" pivot\n",
" .reset_index()\n",
" .melt(id_vars=\"policy_type\", var_name=\"month\", value_name=\"number_of_complaints\")\n",
" .sort_values([\"month\", \"policy_type\"])\n",
" .reset_index(drop=True)\n",
")\n",
" \n",
"print(\"\\n=== Complaints by Policy Type & Month (Long Format) ===\")\n",
"print(long_df.to_string(index=False))\n",
" \n",
" \n",
"# ─────────────────────────────────────────────────────────────\n",
"# 10. SAVE\n",
"# ─────────────────────────────────────────────────────────────\n",
"df.to_csv(\"marketing_customer_clean.csv\", index=False)\n",
"long_df.to_csv(\"complaints_long_format.csv\", index=False)\n",
"print(\"\\nSaved: marketing_customer_clean.csv & complaints_long_format.csv\")\n",
" \n"
]
},
{
Expand Down Expand Up @@ -79,7 +238,27 @@
},
"outputs": [],
"source": [
"# Your code goes here"
"import pandas as pd\n",
"\n",
"df = pd.read_csv(\"https://raw.githubusercontent.com/data-bootcamp-v4/data/main/marketing_customer_analysis_clean.csv\")\n",
"\n",
"pivot1 = df.pivot_table(\n",
" values=\"total_claim_amount\",\n",
" index=\"sales_channel\",\n",
" aggfunc=\"sum\"\n",
").round(2).sort_values(\"total_claim_amount\", ascending=False)\n",
"\n",
"pivot1.columns = [\"total_revenue\"]\n",
"print(pivot1)\n",
"\n",
"pivot2 = df.pivot_table(\n",
" values=\"customer_lifetime_value\",\n",
" index=\"education\",\n",
" columns=\"gender\",\n",
" aggfunc=\"mean\"\n",
").round(2)\n",
"\n",
"print(pivot2)"
]
},
{
Expand Down Expand Up @@ -137,7 +316,27 @@
},
"outputs": [],
"source": [
"# Your code goes here"
"import pandas as pd\n",
"\n",
"df = pd.read_csv(\"https://raw.githubusercontent.com/data-bootcamp-v4/data/main/marketing_customer_analysis_clean.csv\")\n",
"\n",
"# Step 1: pivot table (wide format) — sum of complaints by policy type × month\n",
"pivot = df.pivot_table(\n",
" values=\"number_of_open_complaints\",\n",
" index=\"policy_type\",\n",
" columns=\"month\",\n",
" aggfunc=\"sum\"\n",
").round(2)\n",
"\n",
"# Step 2: melt to long format — one row per (policy_type, month) combination\n",
"long_df = pivot.reset_index().melt(\n",
" id_vars=\"policy_type\",\n",
" var_name=\"month\",\n",
" value_name=\"number_of_complaints\"\n",
")\n",
"\n",
"long_df = long_df.sort_values([\"month\", \"policy_type\"]).reset_index(drop=True)\n",
"print(long_df)"
]
}
],
Expand Down