Rewrite

.github/workflows/deploy.yml

@@ -1,43 +1,53 @@
 name: Deploy to GitHub Pages
 
 on:
-  # Allows you to run this workflow manually from the Actions tab on GitHub.
-  workflow_dispatch:
-  # Trigger workflow when one of the data files was changed (check if separate jobs of gather-data each trigger this workflow separately)
-  #  push:
-  #    paths:
-  #      - '**/*.json'
+    # Allows you to run this workflow manually from the Actions tab on GitHub.
+    workflow_dispatch:
 
 # Allow this job to clone the repo and create a page deployment
 permissions:
-  contents: read
-  pages: write
-  id-token: write
+    contents: read
+    pages: write
+    id-token: write
 
 jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout your repository using git
-        uses: actions/checkout@v4
-      - name: Setup Bun
-        uses: oven-sh/setup-bun@v2
-      - name: Install dependencies
-        run: bun install --frozen-lockfile
-      - name: Install, build, and upload your site
-        uses: withastro/action@v3
-        with:
-          path: website # The root location of your Astro project inside the repository. (optional)
-          # node-version: 18 # The specific version of Node that should be used to build your site. Defaults to 18. (optional)
-          package-manager: bun@latest # The Node package manager that should be used to install dependencies and build your site. Automatically detected based on your lockfile. (optional)
-
-  deploy:
-    needs: build
-    runs-on: ubuntu-latest
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-    steps:
-      - name: Deploy to GitHub Pages
-        id: deployment
-        uses: actions/deploy-pages@v4
+    build:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout your repository using git
+              uses: actions/checkout@v4
+
+            - name: Install Bun
+              uses: oven-sh/setup-bun@v1
+              with:
+                   bun-version: latest
+
+            - name: Install Rust
+              uses: actions-rust-lang/setup-rust-toolchain@v1
+              with:
+                  toolchain: nightly
+                  target: wasm32-unknown-unknown
+
+            - name: Install wasm-pack
+              run: bun install -g wasm-pack
+
+            - name: Build data-wasm
+              run: make wasm 
+
+            - name: Install, build, and upload your site
+              uses: withastro/action@v3
+              with:
+                  path: website # The root location of your Astro project inside the repository. (optional)
+                  # node-version: 18 # The specific version of Node that should be used to build your site. Defaults to 18. (optional)
+                  package-manager: bun@latest # The Node package manager that should be used to install dependencies and build your site. Automatically detected based on your lockfile. (optional)
+
+    deploy:
+        needs: build
+        runs-on: ubuntu-latest
+        environment:
+            name: github-pages
+            url: ${{ steps.deployment.outputs.page_url }}
+        steps:
+            - name: Deploy to GitHub Pages
+              id: deployment
+              uses: actions/deploy-pages@v4

.gitignore

@@ -113,7 +113,6 @@ web_modules/
 # Next.js build output
 
 .next
-out
 
 # Nuxt.js build / generate output
 

.vscode/settings.json

@@ -2,6 +2,10 @@
 	"files.exclude": {
 		"obsidian-releases": true,
 		"choosealicense.com": false,
-		"pluginRepos": true
-	}
+		"pluginRepos": true,
+		"data/out/plugin-repos/**": true
+	},
+	"rust-analyzer.files.exclude": [
+		"data/out/**"
+	]
 }

Makefile

@@ -0,0 +1,19 @@
+wasm:
+	cd data-wasm && make
+
+data:
+	cd data && make
+
+format:
+	cd data && make format
+	cd data-lib && make format
+	cd data-wasm && make format
+	cd website && bun run format
+
+lint:
+	cd data && make lint
+	cd data-lib && make lint
+	cd data-wasm && make lint
+
+submodule-update:
+	git submodule update --init --remote

README.md

@@ -4,6 +4,65 @@ A stats website for the Obsidian Eco-System.
 
 https://www.moritzjung.dev/obsidian-stats/
 
+## Project Structure
+
+The project is split into 4 parts.
+
+1. The website located in `website`
+2. A shared Rust library in `data-lib`
+3. The data collection code in `data`
+4. The data analysis code used for the website in `data-wasm`
+
+All collected data is persisted in the `data/out` folder and tracked in Git, except for cloned plugin repos.
+
+The project uses Makefiles that sometimes think targets are up to date. In that case you can add the `-B` flag to the make command.
+
+### How to Build
+
+Requirements:
+
+1. [Bun](https://bun.com/)
+2. [Latest Rust Compiler](https://rustup.rs/)
+3. [wasm-pack](https://github.com/drager/wasm-pack)
+4. Probably be on Linux, idk
+
+To test and build the website locally you first need to build the `data-wasm` folder. 
+For this you can run `make wasm` in the repo root.
+
+After this you can navigate to the `website` folder and install dependencies with `bun i`.
+Then you can run either the dev server with `bun run dev` or build the website via `bun run build`.
+
+### How to Run Data Collection
+
+Requirements:
+
+1. [Latest Rust Compiler](https://rustup.rs/)
+2. Probably Linux, idk
+3. 10+ minutes of time
+
+Depending on your goal, the first thing you might want to run is `make submodule-update`.
+
+After that you can run the data collection via `make data`.
+
+The data collection consists of multiple phases:
+
+1. Theme Data
+    1. Read theme data from the `obsidian-releases` repo
+    2. Build and save theme data to `data/out/theme-data`
+2. Plugin Data
+    1. Read plugin data from the `obsidian-releases` repo, including download data
+    2. Build plugin, download, and version data
+    3. Save that data to `data/out/plugin-data`
+3. Clone Plugin Repos to `data/out/plugin-repos`
+4. Extract extra data from plugin repos to `data/out/plugin-repo-data`
+5. Process licenses from `choosealicense.com` to `data/out/licenses.json`
+6. Obsidian Releases Data
+    1. Get release data from GitHub releases, process, and save to `data/out/releases-github-raw`. This data is incremental.
+    2. Interpolate download data and save to `data/out/releases-github-interpolated`
+    3. Get the Obsidian changelog, process and save to `data/out/releases-changelog`
+
+Currently you select which steps to run by commenting out the steps that you don't want in `data/src/main.rs`.
+
 ## Credits
 
 I want to thank the following people:

data-lib/.gitignore

@@ -0,0 +1 @@
+/target