Unify X and layers #1707

flying-sheep · 2024-10-09T07:33:11Z

Closes Unification of X and layers #244, closes concat when some values of X are None #678
Tests added
Release note added (or unnecessary)

TODO:

decide on dtype arg: just remove? or ignore+warning? (unless we already do that, then remove)
backed mode: replace with Dask?
concat when some values of X are None #678

codecov · 2024-10-09T08:06:18Z

❌ 156 Tests Failed:

Tests completed	Failed	Passed	Skipped
6826	156	6670	1186

View the top 3 failed test(s) by shortest run time

tests.test_deprecations::test_dtype_warning

Stack Traces | 0.002s run time

#x1B[0m#x1B[94mdef#x1B[39;49;00m#x1B[90m #x1B[39;49;00m#x1B[92mtest_dtype_warning#x1B[39;49;00m():#x1B[90m#x1B[39;49;00m
        #x1B[90m# Tests a warning is thrown#x1B[39;49;00m#x1B[90m#x1B[39;49;00m
        #x1B[94mwith#x1B[39;49;00m pytest.warns(#x1B[96mFutureWarning#x1B[39;49;00m):#x1B[90m#x1B[39;49;00m
>           a = AnnData(np.ones((#x1B[94m3#x1B[39;49;00m, #x1B[94m3#x1B[39;49;00m)), dtype=np.float32)#x1B[90m#x1B[39;49;00m

#x1B[1m#x1B[31mtests/test_deprecations.py#x1B[0m:81: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

args_all = (<[AttributeError("'AnnData' object has no attribute '_is_view'") raised in repr()] AnnData object at 0x7fcca939e090>, array([[1., 1., 1.],
       [1., 1., 1.],
       [1., 1., 1.]]))
kw = {'dtype': <class 'numpy.float32'>}

    #x1B[0m#x1B[37m@wraps#x1B[39;49;00m(fn)#x1B[90m#x1B[39;49;00m
    #x1B[94mdef#x1B[39;49;00m#x1B[90m #x1B[39;49;00m#x1B[92mfn_compatible#x1B[39;49;00m(*args_all: P.args, **kw: P.kwargs) -> R:#x1B[90m#x1B[39;49;00m
        #x1B[94mif#x1B[39;49;00m #x1B[96mlen#x1B[39;49;00m(args_all) <= n_positional:#x1B[90m#x1B[39;49;00m
>           #x1B[94mreturn#x1B[39;49;00m fn(*args_all, **kw)#x1B[90m#x1B[39;49;00m
#x1B[1m#x1B[31mE           TypeError: AnnData.__init__() got an unexpected keyword argument 'dtype'#x1B[0m

#x1B[1m#x1B[31m../../../..../anndata/88DSKUwu/hatch-test.pre/lib/python3.13.../site-packages/legacy_api_wrap/__init__.py#x1B[0m:82: TypeError

#x1B[33mDuring handling of the above exception, another exception occurred:#x1B[0m

    #x1B[0m#x1B[94mdef#x1B[39;49;00m#x1B[90m #x1B[39;49;00m#x1B[92mtest_dtype_warning#x1B[39;49;00m():#x1B[90m#x1B[39;49;00m
        #x1B[90m# Tests a warning is thrown#x1B[39;49;00m#x1B[90m#x1B[39;49;00m
>       #x1B[94mwith#x1B[39;49;00m pytest.warns(#x1B[96mFutureWarning#x1B[39;49;00m):#x1B[90m#x1B[39;49;00m
#x1B[1m#x1B[31mE       Failed: DID NOT WARN. No warnings of type (<class 'FutureWarning'>,) were emitted.#x1B[0m
#x1B[1m#x1B[31mE        Emitted warnings: [].#x1B[0m

#x1B[1m#x1B[31mtests/test_deprecations.py#x1B[0m:80: Failed

tests.test_views::test_set_scalar_subset_X[np_array-single_int_subset]

Stack Traces | 0.003s run time

matrix_type = <function asarray at 0x7fcce24ebce0>
subset_func = <function single_int_subset at 0x7fccd7106700>

    #x1B[0m#x1B[37m@IGNORE_SPARSE_EFFICIENCY_WARNING#x1B[39;49;00m#x1B[90m#x1B[39;49;00m
    #x1B[94mdef#x1B[39;49;00m#x1B[90m #x1B[39;49;00m#x1B[92mtest_set_scalar_subset_X#x1B[39;49;00m(matrix_type, subset_func):#x1B[90m#x1B[39;49;00m
        adata = ad.AnnData(matrix_type(np.zeros((#x1B[94m10#x1B[39;49;00m, #x1B[94m10#x1B[39;49;00m))))#x1B[90m#x1B[39;49;00m
        orig_X_val = adata.X.copy()#x1B[90m#x1B[39;49;00m
        subset_idx = subset_func(adata.obs_names)#x1B[90m#x1B[39;49;00m
    #x1B[90m#x1B[39;49;00m
        adata_subset = adata[subset_idx, :]#x1B[90m#x1B[39;49;00m
    #x1B[90m#x1B[39;49;00m
>       adata_subset.X = #x1B[94m1#x1B[39;49;00m#x1B[90m#x1B[39;49;00m

#x1B[1m#x1B[31mtests/test_views.py#x1B[0m:390: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
#x1B[1m#x1B[.../anndata/_core/anndata.py#x1B[0m:595: in X
    #x1B[0m#x1B[96mself#x1B[39;49;00m.layers[#x1B[94mNone#x1B[39;49;00m] = value#x1B[90m#x1B[39;49;00m
#x1B[1m#x1B[.../anndata/_core/aligned_mapping.py#x1B[0m:174: in __setitem__
    #x1B[0mvalue = #x1B[96mself#x1B[39;49;00m._validate_value(value, key)  #x1B[90m# Validate before mutating#x1B[39;49;00m#x1B[90m#x1B[39;49;00m
#x1B[1m#x1B[.../anndata/_core/aligned_mapping.py#x1B[0m:86: in _validate_value
    #x1B[0m#x1B[94mif#x1B[39;49;00m #x1B[96mself#x1B[39;49;00m.parent.shape[axis] == axis_len(val, i):#x1B[90m#x1B[39;49;00m
#x1B[1m#x1B[31m../../../..../uv/python/cpython-3.13.7-linux-x86_64-gnu/lib/python3.13/functools.py#x1B[0m:934: in wrapper
    #x1B[0m#x1B[94mreturn#x1B[39;49;00m dispatch(args[#x1B[94m0#x1B[39;49;00m].#x1B[91m__class__#x1B[39;49;00m)(*args, **kw)#x1B[90m#x1B[39;49;00m
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

x = 1, axis = 0

    #x1B[0m#x1B[37m@singledispatch#x1B[39;49;00m#x1B[90m#x1B[39;49;00m
    #x1B[94mdef#x1B[39;49;00m#x1B[90m #x1B[39;49;00m#x1B[92maxis_len#x1B[39;49;00m(x, axis: Literal[#x1B[94m0#x1B[39;49;00m, #x1B[94m1#x1B[39;49;00m]) -> #x1B[96mint#x1B[39;49;00m | #x1B[94mNone#x1B[39;49;00m:#x1B[90m#x1B[39;49;00m
    #x1B[90m    #x1B[39;49;00m#x1B[33m"""\#x1B[39;49;00m
    #x1B[33m    Return the size of an array in dimension `axis`.#x1B[39;49;00m
    #x1B[33m#x1B[39;49;00m
    #x1B[33m    Returns None if `x` is an awkward array with variable length in the requested dimension.#x1B[39;49;00m
    #x1B[33m    """#x1B[39;49;00m#x1B[90m#x1B[39;49;00m
>       #x1B[94mreturn#x1B[39;49;00m x.shape[axis]#x1B[90m#x1B[39;49;00m
#x1B[1m#x1B[31mE       AttributeError: 'int' object has no attribute 'shape'#x1B[0m

#x1B[1m#x1B[31msrc/anndata/utils.py#x1B[0m:118: AttributeError

tests.test_views::test_set_scalar_subset_X[np_array-slice_int_subset]

Stack Traces | 0.003s run time

matrix_type = <function asarray at 0x7fcce24ebce0>
subset_func = <function slice_int_subset at 0x7fccd7106660>

    #x1B[0m#x1B[37m@IGNORE_SPARSE_EFFICIENCY_WARNING#x1B[39;49;00m#x1B[90m#x1B[39;49;00m
    #x1B[94mdef#x1B[39;49;00m#x1B[90m #x1B[39;49;00m#x1B[92mtest_set_scalar_subset_X#x1B[39;49;00m(matrix_type, subset_func):#x1B[90m#x1B[39;49;00m
        adata = ad.AnnData(matrix_type(np.zeros((#x1B[94m10#x1B[39;49;00m, #x1B[94m10#x1B[39;49;00m))))#x1B[90m#x1B[39;49;00m
        orig_X_val = adata.X.copy()#x1B[90m#x1B[39;49;00m
        subset_idx = subset_func(adata.obs_names)#x1B[90m#x1B[39;49;00m
    #x1B[90m#x1B[39;49;00m
        adata_subset = adata[subset_idx, :]#x1B[90m#x1B[39;49;00m
    #x1B[90m#x1B[39;49;00m
>       adata_subset.X = #x1B[94m1#x1B[39;49;00m#x1B[90m#x1B[39;49;00m

#x1B[1m#x1B[31mtests/test_views.py#x1B[0m:390: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
#x1B[1m#x1B[.../anndata/_core/anndata.py#x1B[0m:595: in X
    #x1B[0m#x1B[96mself#x1B[39;49;00m.layers[#x1B[94mNone#x1B[39;49;00m] = value#x1B[90m#x1B[39;49;00m
#x1B[1m#x1B[.../anndata/_core/aligned_mapping.py#x1B[0m:174: in __setitem__
    #x1B[0mvalue = #x1B[96mself#x1B[39;49;00m._validate_value(value, key)  #x1B[90m# Validate before mutating#x1B[39;49;00m#x1B[90m#x1B[39;49;00m
#x1B[1m#x1B[.../anndata/_core/aligned_mapping.py#x1B[0m:86: in _validate_value
    #x1B[0m#x1B[94mif#x1B[39;49;00m #x1B[96mself#x1B[39;49;00m.parent.shape[axis] == axis_len(val, i):#x1B[90m#x1B[39;49;00m
#x1B[1m#x1B[31m../../../..../uv/python/cpython-3.13.7-linux-x86_64-gnu/lib/python3.13/functools.py#x1B[0m:934: in wrapper
    #x1B[0m#x1B[94mreturn#x1B[39;49;00m dispatch(args[#x1B[94m0#x1B[39;49;00m].#x1B[91m__class__#x1B[39;49;00m)(*args, **kw)#x1B[90m#x1B[39;49;00m
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

x = 1, axis = 0

    #x1B[0m#x1B[37m@singledispatch#x1B[39;49;00m#x1B[90m#x1B[39;49;00m
    #x1B[94mdef#x1B[39;49;00m#x1B[90m #x1B[39;49;00m#x1B[92maxis_len#x1B[39;49;00m(x, axis: Literal[#x1B[94m0#x1B[39;49;00m, #x1B[94m1#x1B[39;49;00m]) -> #x1B[96mint#x1B[39;49;00m | #x1B[94mNone#x1B[39;49;00m:#x1B[90m#x1B[39;49;00m
    #x1B[90m    #x1B[39;49;00m#x1B[33m"""\#x1B[39;49;00m
    #x1B[33m    Return the size of an array in dimension `axis`.#x1B[39;49;00m
    #x1B[33m#x1B[39;49;00m
    #x1B[33m    Returns None if `x` is an awkward array with variable length in the requested dimension.#x1B[39;49;00m
    #x1B[33m    """#x1B[39;49;00m#x1B[90m#x1B[39;49;00m
>       #x1B[94mreturn#x1B[39;49;00m x.shape[axis]#x1B[90m#x1B[39;49;00m
#x1B[1m#x1B[31mE       AttributeError: 'int' object has no attribute 'shape'#x1B[0m

#x1B[1m#x1B[31msrc/anndata/utils.py#x1B[0m:118: AttributeError

To view more test analytics, go to the Test Analytics Dashboard
_{📋 Got 3 mins? Take this short survey to help us improve Test Analytics.}

ilan-gold · 2025-08-22T15:59:00Z

So re: the backed question, I think the answer is not to get rid of it, but to clarify its behavior. To do that, I think we should merge #1927 first, which may include a breaking change. If we stop allowing overriding of on-disk stores (which currently relies on inheritance form scipy.sparse private methods), then this question becomes much more simplified.

The next step after #1927 would probably be to simplify it. From what I can tell, the reaosn we need to track if the file is open or not is because if writing. If everything were purely read-only, I think it would be less of an issue.

My vision there would be in "backed" mode (after the two above points are implemented), we would simply take the X attribute on-disk, and read it in using ad.io.sparse_dataset with mode="r" and call it a day. That's how layers and other elements handle sparse_dataset so X shouldn't be different

ilan-gold · 2025-08-27T15:20:32Z

src/anndata/_core/anndata.py


    @X.setter
-    def X(self, value: XDataType | None):  # noqa: PLR0912
-        if value is None:


I think a lot of this deleted logic contains things that layers can't currently do. For example, overriding the backing X; currently, if you try to override a part of a view of a layers, it will just actualize and not override the old data. But if you do adata_view.X = 1 it will fill ones in to the parent AnnData object. I'm not sure what the history is there, but it accounts for a pretty big discrepancy in functionality

Co-authored-by: Philipp A. <[email protected]>

WIP

8143e07

flying-sheep changed the title ~~WIP~~ Unify X and layers Oct 9, 2024

flying-sheep added 2 commits October 10, 2024 16:20

make all test modules import

2bfdd7a

some misc fixes

2d13618

ilan-gold assigned flying-sheep Oct 17, 2024

flying-sheep added 4 commits October 25, 2024 16:52

Merge branch 'main' into x-layers-unification

b8f91c0

small fixes

f12f4d7

fix write_anndata

95e8f1d

some more fixes

a6555f8

flying-sheep added this to the 0.12.0 milestone Nov 5, 2024

Zethson mentioned this pull request Jul 12, 2025

Proposal: Stop using AnnData.raw chanzuckerberg/single-cell-curation#1304

Open

flying-sheep added 5 commits July 15, 2025 10:34

Merge branch 'main' into x-layers-unification

56252a8

make setting X to None equivalent to deleting it

77024ba

handle X from layers

208b8bb

simplify copy

fca1a87

delegate transpose to layers

5ac31c4

ilan-gold modified the milestones: 0.12.0, 0.12.1, 0.13.0 Jul 16, 2025

ilan-gold mentioned this pull request Jul 29, 2025

anndata 0.2.0 on-disk format #2060

Open

4 tasks

Merge branch 'main' into x-layers-unification

1d0151b

ilan-gold reviewed Aug 27, 2025

View reviewed changes

ilan-gold and others added 2 commits August 29, 2025 13:35

fix: most immutable setting situtations (#2095)

3756974

Co-authored-by: Philipp A. <[email protected]>

make them falsy

28436ec

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Unify X and layers #1707

Unify X and layers #1707

Uh oh!

flying-sheep commented Oct 9, 2024 •

edited

Loading

Uh oh!

codecov bot commented Oct 9, 2024 •

edited

Loading

Uh oh!

ilan-gold commented Aug 22, 2025 •

edited

Loading

Uh oh!

ilan-gold Aug 27, 2025 •

edited

Loading

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

2 participants

Unify X and layers #1707

Are you sure you want to change the base?

Unify X and layers #1707

Uh oh!

Conversation

flying-sheep commented Oct 9, 2024 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

codecov bot commented Oct 9, 2024 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

❌ 156 Tests Failed:

Uh oh!

ilan-gold commented Aug 22, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

ilan-gold Aug 27, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

2 participants

flying-sheep commented Oct 9, 2024 •

edited

Loading

codecov bot commented Oct 9, 2024 •

edited

Loading

ilan-gold commented Aug 22, 2025 •

edited

Loading

ilan-gold Aug 27, 2025 •

edited

Loading