Pandas-related incompatibility on python 3.12 with newer versions of pandas #636
Description
Describe the bug
There is an incompatibility that was turned up by some of the new unit tests that were added in PR #379 related to functionality involving pandas dataframes. The two tests which gave rise to this error were test_e2e_mirror_rb in test_ibmqexperiment.py and test_dataframe_conversion in test_protocols.py. For the mirror rb test I've temporarily commented out the offending dataframe conversion, and I'm having pytest skip the other test entirely.
To Reproduce
The quickest way to reproduce this error is to run the two tests in question in an environment with pandas 2.3.1 installed while using python 3.12 (newer versions of python likely have the same problem).
Environment (please complete the following information):
- pyGSTi version develop
- python version 3.12
Additional context
I spent some time seeing if I could quickly track this down, and I got as far as finding some clues online that suggest that what might be happening is that a change made in python 3.12 involving slice objects (making them hashable objects) might be interacting in an unexpected way with pyGSTi dataframe utility code.
Below is the traceback captured by pytest for the test_e2e_mirror_rb test failure.
=============================================================================== FAILURES ================================================================================
________________________________________________________________ IBMQExperimentTester.test_e2e_mirror_rb ________________________________________________________________
self = Index(['polarization', 'success_counts', 'success_probabilities',
'total_counts', 'two_q_gate_count'],
dtype='object'), key = slice(None, None, None)
def get_loc(self, key):
"""
Get integer location, slice or boolean mask for requested label.
Parameters
----------
key : label
Returns
-------
int if unique index, slice if monotonic index, else mask
Examples
--------
>>> unique_index = pd.Index(list('abc'))
>>> unique_index.get_loc('b')
1
>>> monotonic_index = pd.Index(list('abbc'))
>>> monotonic_index.get_loc('b')
slice(1, 3, None)
>>> non_monotonic_index = pd.Index(list('abcb'))
>>> non_monotonic_index.get_loc('b')
array([False, True, False, True])
"""
casted_key = self._maybe_cast_indexer(key)
try:
> return self._engine.get_loc(casted_key)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
..\..\..\..\.venv312\Lib\site-packages\pandas\core\indexes\base.py:3812:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
pandas/_libs/index.pyx:167: in pandas._libs.index.IndexEngine.get_loc
???
pandas/_libs/index.pyx:196: in pandas._libs.index.IndexEngine.get_loc
???
pandas/_libs/hashtable_class_helper.pxi:7088: in pandas._libs.hashtable.PyObjectHashTable.get_item
???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> ???
E KeyError: slice(None, None, None)
pandas/_libs/hashtable_class_helper.pxi:7096: KeyError
During handling of the above exception, another exception occurred:
self = <test.unit.extras.ibmq.test_ibmqexperiment.IBMQExperimentTester object at 0x000001397F57CAD0>
def test_e2e_mirror_rb(self):
# Have to do int(i) because variable is of wrong type. Well, maybe.
edges = [(int(i), int(j)) for (i,j) in list(self.backend.coupling_map.get_edges())]
qubit_labels = [i for i in range(self.backend.num_qubits)]
num_qubits = self.backend.num_qubits
two_qubit_gate = 'Gcphase'
gate_names = ['Gc{}'.format(i) for i in range(24)] + [two_qubit_gate,]
availability = {two_qubit_gate: edges}
pspec = pygsti.processors.QubitProcessorSpec(num_qubits, gate_names, availability=availability,
qubit_labels=qubit_labels)
clifford_compilations = {'absolute': pygsti.processors.CliffordCompilationRules.create_standard(pspec, verbosity=0)}
#mirror rb design parameters
qubit_labels = [i for i in range(self.backend.num_qubits)]
widths = [1, 2, 3, 4]
depths = [0, 10]
qubits = {w: tuple(qubit_labels[0:w]) for w in widths}
circuits_per_shape = 5
xi = {w:1/4 for w in widths}
if 1 in widths: xi[1] = 0 # No two-qubit gates in one-qubit circuits.
#build mirror RB design
edesigns = {}
for w in widths:
key = str(w)+ '-' 'random'
edesigns[key] = RMCDesign(pspec, depths, circuits_per_shape, clifford_compilations=clifford_compilations,
qubit_labels=qubits[w], sampler='edgegrab', samplerargs=[xi[w],])
for w in widths:
key = str(w)+ '-' 'periodic'
# xi has a different meaning in the PMC design --> twice what it is in RMC design
edesigns[key] = PMCDesign(pspec, depths, circuits_per_shape, clifford_compilations=clifford_compilations,
qubit_labels=qubits[w], sampler='edgegrab', samplerargs=[xi[w]/2,])
combined_edesign = pygsti.protocols.CombinedExperimentDesign(edesigns)
exp = ibmq.IBMQExperiment(combined_edesign, pspec, checkpoint_override=True)
exp.transpile(self.backend)
exp.submit(self.backend)
exp.monitor()
exp.retrieve_results()
data = exp.data
# The summary statistics to calculate for each circuit.
statistics = ['polarization', 'success_probabilities', 'success_counts', 'total_counts', 'two_q_gate_count']
stats_generator = pygsti.protocols.SimpleRunner(ByDepthSummaryStatistics(statistics_to_compute=statistics))
# Computes the summary statistics for each circuit
results = stats_generator.run(data)
# Turns the results into a data frame.
> df = results.to_dataframe('ValueName', drop_columns=['ProtocolName','ProtocolType'])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
test_ibmqexperiment.py:123:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
..\..\..\..\pygsti\protocols\protocol.py:3053: in to_dataframe
return _process_dataframe(df, pivot_valuename, pivot_value, drop_columns)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
..\..\..\..\pygsti\tools\dataframetools.py:49: in _process_dataframe
df = _reset_index(df_unstacked)
^^^^^^^^^^^^^^^^^^^^^^^^^^
..\..\..\..\pygsti\tools\dataframetools.py:27: in _reset_index
return _pd.merge(index_df, df, left_index=True, right_index=True)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
..\..\..\..\.venv312\Lib\site-packages\pandas\core\reshape\merge.py:184: in merge
return op.get_result(copy=copy)
^^^^^^^^^^^^^^^^^^^^^^^^
..\..\..\..\.venv312\Lib\site-packages\pandas\core\reshape\merge.py:888: in get_result
result = self._reindex_and_concat(
..\..\..\..\.venv312\Lib\site-packages\pandas\core\reshape\merge.py:838: in _reindex_and_concat
right = self.right[:]
^^^^^^^^^^^^^
..\..\..\..\.venv312\Lib\site-packages\pandas\core\frame.py:4080: in __getitem__
and key in self.columns
^^^^^^^^^^^^^^^^^^^
..\..\..\..\.venv312\Lib\site-packages\pandas\core\indexes\category.py:368: in __contains__
return contains(self, key, container=self._engine)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
..\..\..\..\.venv312\Lib\site-packages\pandas\core\arrays\categorical.py:230: in contains
loc = cat.categories.get_loc(key)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = Index(['polarization', 'success_counts', 'success_probabilities',
'total_counts', 'two_q_gate_count'],
dtype='object'), key = slice(None, None, None)
def get_loc(self, key):
"""
Get integer location, slice or boolean mask for requested label.
Parameters
----------
key : label
Returns
-------
int if unique index, slice if monotonic index, else mask
Examples
--------
>>> unique_index = pd.Index(list('abc'))
>>> unique_index.get_loc('b')
1
>>> monotonic_index = pd.Index(list('abbc'))
>>> monotonic_index.get_loc('b')
slice(1, 3, None)
>>> non_monotonic_index = pd.Index(list('abcb'))
>>> non_monotonic_index.get_loc('b')
array([False, True, False, True])
"""
casted_key = self._maybe_cast_indexer(key)
try:
return self._engine.get_loc(casted_key)
except KeyError as err:
if isinstance(casted_key, slice) or (
isinstance(casted_key, abc.Iterable)
and any(isinstance(x, slice) for x in casted_key)
):
> raise InvalidIndexError(key)
E pandas.errors.InvalidIndexError: slice(None, None, None)
..\..\..\..\.venv312\Lib\site-packages\pandas\core\indexes\base.py:3818: InvalidIndexError