Skip to content

Commit 8f2f636

Browse files
author
Release Manager
committed
gh-38728: Improve conda setup <!-- ^ Please provide a concise and informative title. --> <!-- ^ Don't put issue numbers in the title, do this in the PR description below. --> <!-- ^ For example, instead of "Fixes #12345" use "Introduce new method to calculate 1 + 2". --> <!-- v Describe your changes below in detail. --> <!-- v Why is this change required? What problem does it solve? --> <!-- v If this PR resolves an open issue, please link to it here. For example, "Fixes #12345". --> A few small quality of life improvements to the conda setup. Notably the environment files under `src` are now moved to the root, as the old env files there were non-functional and only confused users. Relately, removed the outdated,not working and untested instructions to use conda solely to provide the system packages for sage-the-distro. A few other improvements along the way: - Make conda in devcontainer working again by forcing mamba v1 (v2 was released a few days ago and breaks a few things related to the lock files) - Force usage of conda-forge everywhere (mixing of channels is no longer supported) ### 📝 Checklist <!-- Put an `x` in all the boxes that apply. --> - [ ] The title is concise and informative. - [ ] The description explains in detail what this PR is about. - [ ] I have linked a relevant issue or discussion. - [ ] I have created tests covering the changes. - [ ] I have updated the documentation and checked the documentation preview. ### ⌛ Dependencies <!-- List all open PRs that this PR logically depends on. For example, --> <!-- - #12345: short description why this is a dependency --> <!-- - #34567: ... --> URL: #38728 Reported by: Tobias Diez Reviewer(s): Dima Pasechnik, Kwankyu Lee, Tobias Diez, Vincent Macri
2 parents 6fb883a + cf31bde commit 8f2f636

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+3165
-5543
lines changed

‎.devcontainer/onCreate-conda.sh‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,11 @@
22
set -e
33

44
# Create conda environment
5-
conda install mamba -n base -c conda-forge -y
6-
mamba env create --file src/environment-dev-3.11-linux.yml || mamba env update --file src/environment-dev-3.11-linux.yml
5+
conda config --env --add channels conda-forge
6+
conda config --env --set channel_priority strict
7+
conda update -y --all --override-channels -c conda-forge
8+
conda install mamba=1 -n base -y
9+
mamba env create -y --file environment-dev-3.11-linux.yml || mamba env update -y --file environment-dev-3.11-linux.yml
710
conda init bash
811

912
# Build sage

‎.github/workflows/ci-conda.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ jobs:
5959
with:
6060
path: ~/conda_pkgs_dir
6161
key:
62-
${{ runner.os }}-conda-${{ hashFiles('src/environment-3.11.yml') }}
62+
${{ runner.os }}-conda-${{ hashFiles('environment-3.11.yml') }}
6363

6464
- name: Setup Conda environment
6565
uses: conda-incubator/setup-miniconda@v3
@@ -70,7 +70,7 @@ jobs:
7070
channels: conda-forge
7171
channel-priority: true
7272
activate-environment: sage
73-
environment-file: src/${{ matrix.conda-env }}-${{ matrix.python }}-${{ startsWith(matrix.os, 'macos') && (startsWith(runner.arch, 'ARM') && 'macos' || 'macos-x86_64') || 'linux' }}.yml
73+
environment-file: ${{ matrix.conda-env }}-${{ matrix.python }}-${{ startsWith(matrix.os, 'macos') && (startsWith(runner.arch, 'ARM') && 'macos' || 'macos-x86_64') || 'linux' }}.yml
7474

7575
- name: Print Conda environment
7676
shell: bash -l {0}

‎.github/workflows/ci-meson.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ jobs:
3939
with:
4040
path: ~/conda_pkgs_dir
4141
key:
42-
${{ runner.os }}-conda-${{ hashFiles('src/environment-3.11-linux.yml') }}
42+
${{ runner.os }}-conda-${{ hashFiles('environment-3.11-linux.yml') }}
4343

4444
- name: Compiler cache
4545
uses: hendrikmuhs/[email protected]
@@ -55,7 +55,7 @@ jobs:
5555
channels: conda-forge
5656
channel-priority: true
5757
activate-environment: sage
58-
environment-file: src/environment-${{ matrix.python }}-${{ startsWith(matrix.os, 'macos') && (startsWith(runner.arch, 'ARM') && 'macos' || 'macos-x86_64') || 'linux' }}.yml
58+
environment-file: environment-${{ matrix.python }}-${{ startsWith(matrix.os, 'macos') && (startsWith(runner.arch, 'ARM') && 'macos' || 'macos-x86_64') || 'linux' }}.yml
5959

6060
- name: Print Conda environment
6161
shell: bash -l {0}

‎.github/workflows/conda-lock-update.py‎

Lines changed: 34 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,30 +4,53 @@
44
import subprocess
55

66
script_dir = Path(__file__).resolve().parent
7-
root_dir = script_dir / '..' / '..'
7+
root_dir = script_dir / ".." / ".."
88

99
subprocess.run([str(root_dir / "bootstrap-conda")])
1010

1111
platforms = {
1212
"linux-64": "linux",
1313
"linux-aarch64": "linux-aarch64",
1414
"osx-64": "macos-x86_64",
15-
"osx-arm64": "macos"
16-
#"win-64": "win",
15+
"osx-arm64": "macos",
16+
# "win-64": "win",
1717
}
1818
pythons = ["3.9", "3.10", "3.11"]
1919
tags = ["", "-dev"]
20-
sources = ["", "src"]
2120

2221
for platform_key, platform_value in platforms.items():
2322
for python in pythons:
2423
for tag in tags:
25-
for src in sources:
26-
env_file = root_dir / src / f"environment{tag}-{python}.yml"
27-
lock_file = root_dir / src / f"environment{tag}-{python}-{platform_value}"
24+
env_file = root_dir / f"environment{tag}-{python}.yml"
25+
lock_file = root_dir / f"environment{tag}-{python}-{platform_value}"
26+
lock_file_gen = root_dir / f"environment{tag}-{python}-{platform_value}.yml"
2827

29-
if not env_file.exists():
30-
continue
28+
if not env_file.exists():
29+
continue
3130

32-
print(f"Updating lock file for {env_file} at {lock_file}", flush=True)
33-
subprocess.run(["conda-lock", "--channel", "conda-forge", "--kind", "env", "--platform", platform_key, "--file", str(env_file), "--lockfile", str(lock_file), "--filename-template", str(lock_file)])
31+
print(f"Updating lock file for {env_file} at {lock_file_gen}", flush=True)
32+
subprocess.run(
33+
[
34+
"conda-lock",
35+
"--mamba",
36+
"--channel",
37+
"conda-forge",
38+
"--kind",
39+
"env",
40+
"--platform",
41+
platform_key,
42+
"--file",
43+
str(env_file),
44+
"--lockfile",
45+
str(lock_file),
46+
"--filename-template",
47+
str(lock_file),
48+
],
49+
check=True,
50+
)
51+
52+
# Add conda env name to lock file at beginning
53+
with open(lock_file_gen, "r+") as f:
54+
content = f.read()
55+
f.seek(0, 0)
56+
f.write(f"name: sage{tag}\n{content}")

‎.gitignore‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,15 @@
2828
/src/lib/pkgconfig
2929

3030
# Environment files generated by bootstrap-conda.
31-
# The files without Python version are no longer generated
31+
# The files without Python version and in src are no longer generated
3232
# but may still be in users' directories.
3333
/environment.yml
3434
/environment-3.9.yml
3535
/environment-3.10.yml
3636
/environment-3.11.yml
37+
/environment-dev-3.9.yml
38+
/environment-dev-3.10.yml
39+
/environment-dev-3.11.yml
3740
/environment-optional.yml
3841
/environment-optional-3.9.yml
3942
/environment-optional-3.10.yml

‎.gitpod.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ tasks:
77
- name: Setup
88
# Create conda environment, then configure and build sage
99
init: >-
10-
&& mamba env create --file src/environment-dev-3.11-linux.yml --prefix venv
10+
&& mamba env create --file environment-dev-3.11-linux.yml --prefix venv
1111
&& conda config --append envs_dirs $(pwd)
1212
&& conda activate $(pwd)/venv
1313
&& ./bootstrap

‎.vscode/settings.json‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
{
2-
// This settings file is not ignored by git. It should be kept in sync with
3-
// the trac repo.
4-
"python.defaultInterpreterPath": "./venv/bin/python3",
2+
// This settings file is not ignored by git.
53
"files.exclude": {
64
"**/__pycache__": true,
75
"src/**/*.cpp": true,

‎bootstrap‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -227,9 +227,6 @@ save () {
227227
src/doc/en/installation/*.txt \
228228
$(find src/doc/en/reference/spkg -name index.rst -prune -o -maxdepth 1 -name "*.rst" -print) \
229229
environment-3.[89].yml environment-3.1[0-9].yml \
230-
src/environment-3.[89].yml src/environment-3.1[0-9].yml \
231-
environment-optional-3.[89].yml environment-optional-3.1[0-9].yml \
232-
src/environment-optional-3.[89].yml src/environment-optional-3.1[0-9].yml \
233230
src/Pipfile \
234231
src/pyproject.toml \
235232
src/requirements.txt \

‎bootstrap-conda‎

Lines changed: 5 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ unset ENABLE_SYSTEM_SITE_PACKAGES
8484
echo >&2 $0:$LINENO: generate conda environment files
8585

8686
(
87-
echo "name: sage-build"
87+
echo "name: sage"
8888
echo "channels:"
8989
echo " - conda-forge"
9090
echo " - nodefaults"
@@ -106,58 +106,18 @@ echo >&2 $0:$LINENO: generate conda environment files
106106
for pkg in $SAGELIB_SYSTEM_PACKAGES; do
107107
echo " - $pkg"
108108
done
109-
) > src/environment-template.yml
110-
111-
(
112-
cat environment-template.yml
113-
echo " # optional packages"
114-
for pkg in $OPTIONAL_SYSTEM_PACKAGES; do
115-
echo " - $pkg"
116-
done
117-
) > environment-optional-template.yml
109+
) > environment-template.yml
118110

119111
(
120-
sed 's/name: sage/name: sage-dev/' src/environment-template.yml
112+
sed 's/name: sage/name: sage-dev/' environment-template.yml
121113
echo " # Additional dev tools"
122114
echo " - conda-lock"
123115
for pkg in $DEVELOP_SYSTEM_PACKAGES; do
124116
echo " - $pkg"
125117
done
126-
) > src/environment-dev-template.yml
127-
128-
(
129-
cat src/environment-template.yml
130-
echo " # optional packages"
131-
for pkg in $OPTIONAL_SYSTEM_PACKAGES $SAGELIB_OPTIONAL_SYSTEM_PACKAGES; do
132-
echo " - $pkg"
133-
done
134-
) > src/environment-optional-template.yml
135-
136-
(
137-
echo >&4 " - pip:"
138-
echo >&5 " - pip:"
139-
for PKG_BASE in $(sage-package list :standard: :optional: --has-file requirements.txt --no-file distros/conda.txt --no-file src; sage-package list :standard: :optional: --has-file version_requirements.txt --no-file requirements.txt --no-file distros/conda.txt --no-file src); do
140-
eval PKG_SCRIPTS=\$path_$PKG_BASE PKG_TYPE=\$type_$PKG_BASE
141-
SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/requirements.txt
142-
if [ ! -f $SYSTEM_PACKAGES_FILE ]; then
143-
SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/version_requirements.txt
144-
fi
145-
if grep -q SAGERUNTIME $PKG_SCRIPTS/dependencies $PKG_SCRIPTS/dependencies_order_only 2>/dev/null; then
146-
: # cannot install packages that depend on the Sage library
147-
else
148-
case "$PKG_BASE:$PKG_TYPE" in
149-
$DEVELOP_SPKG_PATTERN:*) FD=4;;
150-
*:standard) FD="4 5";;
151-
*) FD=5;;
152-
esac
153-
${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE | while read -r line; do
154-
[ -n "$line" ] && for fd in $FD; do echo >&$fd " - $line"; done
155-
done
156-
fi
157-
done
158-
) 4>> /dev/null 5>> src/environment-optional-template.yml
118+
) > environment-dev-template.yml
159119

160-
for f in environment environment-optional src/environment src/environment-optional src/environment-dev; do
120+
for f in environment environment-dev; do
161121
for python_version in 3.9 3.10 3.11; do
162122
sed -E 's/^( *- *)python *$/\1python='$python_version'/' $f-template.yml > $f-$python_version.yml
163123
done

‎condarc.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,3 @@
33
channel_priority: strict
44
channels:
55
- conda-forge
6-
- defaults

0 commit comments

Comments
 (0)