diff --git a/.envrc b/.envrc new file mode 100644 index 000000000..37ef80f76 --- /dev/null +++ b/.envrc @@ -0,0 +1,8 @@ +# we have checks to account for first time setup +if [ -f ".env" ]; then + dotenv +fi +if [ -f "$HOME/.config/nix/nix.conf" ]; then + echo "Entering nix shell environment." + use flake +fi diff --git a/.gitignore b/.gitignore index b65f0ea25..ac56b857b 100644 --- a/.gitignore +++ b/.gitignore @@ -147,7 +147,7 @@ media python.log # direnv -.envrc +.direnv .local_env .DS_Store diff --git a/docs/nix_based_setup.md b/docs/nix_based_setup.md new file mode 100644 index 000000000..c141a8d63 --- /dev/null +++ b/docs/nix_based_setup.md @@ -0,0 +1,140 @@ +# Boost.org Website + +## Overview + +A Django based website that will power a new Boost website. See the [documentation](./docs/README.md) for more information about maintaining this project. + +Links: + +- https://www.stage.boost.cppalliance.org/ - staging +- https://www.boost.org/ - production + +--- + +## Local Development Setup + +This project uses Python 3.11, Docker, and Docker Compose. + +This document describes how to set up a development environment using Nix, which is a package manager that allows for reproducible builds and development environments, like a better encapsulated declarative cross-platform Homebrew. + +For a basic rundown on Nix, this video could be useful https://www.youtube.com/watch?v=yQwW8dkuHqw + +1. Install the following according to the instructions for your platform if not already installed: + 1. Direnv - https://direnv.net/docs/installation.html (don't install OS packaged version, must be >= 2.35.0) and then configure your shell to add the hook as per the direnv docs. + 2. Docker Engine + * Linux - https://docs.docker.com/engine/install/ + * MacOS - https://orbstack.dev/ or https://github.com/abiosoft/colima ? + * Windows - ? + 3. Just - https://just.systems/man/en/packages.html + 4. Nix - https://nixos.org/download/ (multi-user, y to all options) +2. Clone this https://github.com/boostorg/website-v2.git repository to your machine. +3. cd into the repository directory. +4. In a terminal run `just bootstrap` in the root of the checked out repository to install the necessary development dependencies and generate the .env file. + * This will ask you to log in to your google account to allow permissions for the production database load later. Ask Sam for permissions on the database backup drive with an email address associated with Google. +5. Update the generated .env file with the necessary environment variables. Where you can't retrieve these yourself, you can ask someone for some in #boost-website on the slack server at https://ccplang.slack.com. The minimum that must be set is: + * GITHUB_TOKEN - a personal access token for the GitHub API, from your profile + * STATIC_CONTENT_AWS_ACCESS_KEY_ID - ask for this + * STATIC_CONTENT_AWS_SECRET_ACCESS_KEY - ask for this +5. Run `just setup` to build services, and build the JS and CSS assets. If docker fails with permissions errors, reboot your machine. +6. Run `just load_production_data` to download live data from the backup server. +7. Run `docker compose up` to start the server. + +### Social Login with django-allauth + +Follow these instructions to use the social logins through django-allauth on your local machine. + +See https://testdriven.io/blog/django-social-auth/ for more information. + +#### Github +- Go to https://github.com/settings/applications/new and add a new OAuth application +- Set `http://localhost:8000` as the Homepage URL +- Set `http://localhost:8000/accounts/github/login/callback/` as the Callback URL +- Click whether you want to enable the device flow + - The GitHub screen that registers a new OAuth app +- On completion copy the Client ID and Client Secret to the `.env` file as values of `GITHUB_OAUTH_CLIENT_ID` and `GITHUB_OAUTH_CLIENT_SECRET`. +- Run `direnv allow` and restart your docker containers. + +Setup should be complete and you should be able to see an option to "Use Github" on the sign up page. + +To test the flow including authorizing Github for the Boost account, log into your GitHub account settings and click **Applications** in the left menu. Find the "Boost" authorization and delete it. The next time you log into Boost with this GitHub account, you will have to re-authorize it. + +The 'Authorized OAuth Apps' tab in your GitHub Applications + +This setup process is not something that can currently be automated through terraform because of a lack of relevant Github API endpoints to create Oauth credentials. + +#### Google + +More detailed instructions at: + +https://docs.allauth.org/en/latest/socialaccount/providers/google.html + +1. Update the `.env` file with values for: + 1. `TF_VAR_google_cloud_email` (the email address of your Google Cloud account) + 2. `TF_VAR_google_organization_domain` (usually the domain of your Google Cloud account, e.g. "boost.org" if you will be using a @boost.org email address) + 3. `TF_VAR_google_cloud_project_name` (optional, default: localboostdev) - needs to change if destroyed and a setup is needed within 30 days +2. Run `just development-tofu-init` to initialize tofu. +3. Run `just development-tofu-plan` to confirm the planned changes. +4. Run `just development-tofu-apply` to apply the changes. +5. Go to https://console.developers.google.com/ + 1. Search for the newly created project, named "Boost Development" (ID: localboostdev by default). + 2. Type "credentials" in the search input at the top of the page. + 3. Select "Credentials" under "APIs & Services". + 1. Click "+ CREATE CREDENTIALS" + 2. Select "OAuth Client ID" + 3. Select Application Type: "Web application" + 4. Name: "Boost Development" (arbitrary) + 5. For "Authorized Javascript Origins" use:`http://localhost:8000` + 6. For "Authorized Redirect URIs" use: + * `http://localhost:8000/accounts/google/login/callback/` + * `http://localhost:8000/accounts/google/login/callback/?flowName=GeneralOAuthFlow` + 7. Save +6. From the page that's displayed, update the `.env` file with values for the following: + - `GOOGLE_OAUTH_CLIENT_ID` should be similar to "k235bn2b1l1(...)asdsk.apps.googleusercontent.com" + - `GOOGLE_OAUTH_CLIENT_SECRET` should be similar to "LAJACO(...)KLAI612ANAD" +7. Run `docker compose down && docker compose up` and restart your docker containers. + +Point 5 above can not be automated through terraform because of a lack of relevant Google Cloud API endpoints to create Oauth credentials. + +Setup should be complete and you should be able to see an option to "Use Google" on the sign up page. + +#### Additional Notes on allauth login flows: +**Working locally**: If you need to run through the login flows multiple times, create a superuser so you can log into the admin. Then, log into the admin and delete your "Social Account" from the admin. This will test a fresh connection to GitHub for your logged-in GitHub user. + +## Syncing EmailData Locally (optional) + +To work with mailinglist data locally, the django application expects to be +able to query a copy of the hyperkitty database from HYPERKITTY_DATABASE_NAME. +Then, `just manage sync_mailinglist_stats` management command can be run. + +## Debugging +For local development there is Django Debug Toolbar, and the option to set a debugger. + +In your env: +- Django Debug Toolbar, enabled by default, can be disabled by setting DEBUG_TOOLBAR=False +- IDE Debugging, disabled by default, can be enabled by uncommenting `PYTHONBREAKPOINT` in your .env file. + +### Set Up Pycharm +You can set up your IDE with a new "Python Debug Server" configuration as: + +PyCharm Debugger Settings + +### Debugger Usage +To use the debugger add `breakpoint()` on a line in the code before you want to start debugging and then add breakpoints by clicking on the gutter. The debugger will stop at these point, you can then step/inspect the variables. + + +## Troubleshooting + +### Docker +Keep in mind if there are issues with docker that the host docker daemon on your machine and the docker daemon in the nix setup may not match. It's a good idea to keep both up to date. + +### Direnv +when you switch to the directory you may see direnv exporting a bunch of environment variables as below. + +The installer configures direnv to suppress those but it's a recent configuration option, so may be worth checking for an update if you see them. + +## Disk space +Should you find you're running short on disk space, to delete previous versioned store data you can run `nix-collect-garbage -d`. Reentering the directory will then reinstall all the current dependencies again. It's probably a good idea to run that periodically. + +```shell +direnv: export +ALLOWED_HOSTS +AR +AS... +``` diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..e13c6442b --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1736755442, + "narHash": "sha256-a3MMEY7i/wdF0gb7WFNTn6onzaiMOvwj7OerRVenA8o=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "ef56e777fedaa4da8c66a150081523c5de1e0171", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..0c7cbdd85 --- /dev/null +++ b/flake.nix @@ -0,0 +1,96 @@ +{ + description = "Boost.org development environment."; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, nixpkgs, flake-utils, ... }@inputs: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { + inherit system; + }; + # https://nixos.wiki/wiki/Google_Cloud_SDK + gdk = pkgs.google-cloud-sdk.withExtraComponents( with pkgs.google-cloud-sdk.components; [ + gke-gcloud-auth-plugin + ]); + # Install a Ruby gem from rubygems.org + asciidoctorBoostGem = pkgs.stdenv.mkDerivation rec { + pname = "asciidoctor-boost"; + version = "0.1.7"; + sha = "ce139448812a9848219ce4cdb521c83c16009406a9d16efbc90bb24e94a46c24"; + + src = pkgs.fetchurl { + url = "https://rubygems.org/downloads/${pname}-${version}.gem"; + sha256 = "${sha}"; + }; + dontUnpack = true; + nativeBuildInputs = [ pkgs.ruby ]; + buildPhase = "true"; # Nothing to compile. + installPhase = '' + # Create a temporary gem directory + mkdir -p $out + # Set GEM_HOME to install gems locally under $out. + export GEM_HOME=$out + # Install the gem into GEM_HOME. + ${pkgs.ruby}/bin/gem install ${src} --no-document --ignore-dependencies + ''; + meta = { + description = "Asciidoctor Boost Ruby Gem installed from rubygems.org"; + homepage = "https://rubygems.org/gems/asciidoctor-boost"; + license = "BSL-1.0"; + }; + }; + + in { + devShells.default = pkgs.mkShell { + buildInputs = with pkgs; [ + # general system + # e.g. this could contain docker client if we wanted that to be consistent, + # though we need the daemon on the host anyway so it's redundant + # general project + awscli + gdk + just + opentofu + # frontend + nodejs_22 # matches Dockerfile, due for upgrade? + yarn + # backend + asciidoctor + asciidoctorBoostGem + pre-commit + python311 # matches Dockerfile, due for upgrade? + python311.pkgs.black + python311.pkgs.isort + python311.pkgs.pip-tools + ]; + # Host system installation workflow goes into the bootstrap justfile target. + # Project specific installation and execution workflow should go here. + shellHook = '' + if [ ! -f .git/hooks/pre-commit ]; then + pre-commit install + fi + if [ ! -d .venv ]; then + python3.11 -m venv .venv + . .venv/bin/activate + pip install -r requirements.txt -r requirements-dev.txt + else + . .venv/bin/activate + fi + if [ ! -f .env ]; then + cp env.template .env + echo ".env created, you should update its contents" + fi + # google cloud login + gcloud auth list --format="value(account)" | grep -q . || { + echo "Not logged in. Running gcloud auth login..." + gcloud auth login + } + ''; + }; + } + ); +} diff --git a/justfile b/justfile index 5061e0d13..c51f962c9 100644 --- a/justfile +++ b/justfile @@ -12,13 +12,43 @@ ENV_FILE := ".env" # ---- @bootstrap: ## installs/updates all dependencies - #!/usr/bin/env bash - set -euo pipefail - if [ ! -f "{{ENV_FILE}}" ]; then - echo "{{ENV_FILE}} created" - cp env.template {{ENV_FILE}} + command -v direnv >/dev/null 2>&1 || { echo >&2 "Direnv is required but not installed. see: https://direnv.net/docs/installation.html - Aborting."; exit 1; } + command -v nix >/dev/null 2>&1 || { echo >&2 "Nix is required but not installed. see: https://nixos.org/download.html - Aborting."; exit 1; } + command -v just >/dev/null 2>&1 || { echo >&2 "Just is required but not installed. see: https://just.systems/man/en/packages.html - Aborting."; exit 1; } + command -v docker >/dev/null 2>&1 || { echo >&2 "Docker is required but not installed. see: docs for links - Aborting."; exit 1; } + + shell_name=$(basename "$SHELL") && \ + echo $shell_name && \ + if [ "$shell_name" = "zsh" ] && command -v zsh >/dev/null; then \ + zsh -i -c 'echo ${precmd_functions} | grep -q _direnv_hook' || { echo "❌ direnv hook is NOT installed in Zsh"; exit 1; }; \ + elif ([ "$shell_name" = "pwsh" ] || [ "$shell_name" = "powershell" ]) && command -v "$shell_name" >/dev/null; then \ + "$shell_name" -NoProfile -Command '$function:prompt.ToString() | grep -q direnv' || { echo "❌ direnv hook is NOT installed in PowerShell"; exit 1; }; \ + else \ + echo "ℹ️ Unsupported shell for checking direnv hook: $shell_name. Ensure you have the direnv shell hook eval set up correctly if there are problems."; \ fi - docker compose --file {{COMPOSE_FILE}} build --force-rm + + if [ ! -d $HOME/.config/direnv/direnv.toml ]; then \ + mkdir -p $HOME/.config/direnv; \ + printf "[global]\nhide_env_diff = true\nload_dotenv = true\n" > $HOME/.config/direnv/direnv.toml; \ + fi + if [ ! -d $HOME/.config/nix ]; then \ + mkdir -p $HOME/.config/nix; \ + printf "experimental-features = nix-command flakes\n" > $HOME/.config/nix/nix.conf; \ + fi + # check if the docker group exists, create if not + if [ ! $(getent group docker) ]; then \ + echo "ℹ️ Adding docker group..."; \ + sudo groupadd docker; \ + fi + + # check if user is in docker group, add if not + if [ $(id -Gn | grep -c docker) -eq 0 ]; then \ + echo "ℹ️ Adding docker group"; \ + sudo usermod -aG docker $USER; \ + echo "ℹ️ Added docker user. Please close the shell and open a new one."; \ + fi + echo "Bootstrapping complete, update your .env and run 'just setup'" + echo "If you have issues with docker permissions running just setup try restarting your machine." @rebuild: ## rebuilds containers docker compose kill @@ -31,7 +61,7 @@ ENV_FILE := ".env" @build: ## builds containers docker compose pull - DOCKER_BUILDKIT=1 docker compose build + docker compose build @cibuild: ## invoked by continuous integration servers to run tests python -m pytest @@ -49,6 +79,8 @@ alias shell := console @setup: ## sets up a project to be used for the first time docker compose --file {{COMPOSE_FILE}} build --force-rm docker compose --file docker-compose.yml run --rm web python manage.py migrate --noinput + npm install + npm run build @test_pytest: ## runs pytest -docker compose run --rm -e DEBUG_TOOLBAR="False" web pytest -s --create-db diff --git a/scripts/load_production_data.sh b/scripts/load_production_data.sh index 9ac5f9c51..fe5018f94 100755 --- a/scripts/load_production_data.sh +++ b/scripts/load_production_data.sh @@ -5,11 +5,12 @@ source .env download_media_file() { # download all files from the PROD_MEDIA_CONTENT bucket and copy to Docker container - [ -z "$PROD_MEDIA_CONTENT_AWS_ACCESS_KEY_ID" ] && { + # todo: remove the changeme check and remove 'changeme' as the default, use nothing instead + [[ -z "$PROD_MEDIA_CONTENT_AWS_ACCESS_KEY_ID" || "$PROD_MEDIA_CONTENT_AWS_ACCESS_KEY_ID" == "changeme" ]] && { echo "Error: PROD_MEDIA_CONTENT_AWS_ACCESS_KEY_ID not set in .env"; return 1; } - [ -z "$PROD_MEDIA_CONTENT_AWS_SECRET_ACCESS_KEY" ] && { + [[ -z "$PROD_MEDIA_CONTENT_AWS_SECRET_ACCESS_KEY" || "$PROD_MEDIA_CONTENT_AWS_SECRET_ACCESS_KEY" = "changeme" ]] && { echo "Error: PROD_MEDIA_CONTENT_AWS_SECRET_ACCESS_KEY not set in .env"; return 1; } @@ -143,4 +144,4 @@ npm install npm run build echo "Run: 'docker compose up -d' to restart your services" echo "If you get an error related to profile images when loading the site, clear all cookies and try again" -echo "You should now able to log into the admin interface with username: 'superadmin' and password: 'foobarone'" +echo "You should now able to log into the admin interface with username: 'superadmin@boost.org' and password: 'foobarone'"