Revert "Merge pull request #7546 from jenshnielsen/disable_new_export"

docs/changes/newsfragments/7546.breaking

@@ -0,0 +1,7 @@
+The functionality to export qcodes datasets to XArray/netcdf4 has been significantly for datasets where the shape of the data is known from metadata e.g. using dond or similar:
+
+* The memory overhead of exporting the data has been reduced.
+* The coordinate order is maintained such that data is stored in the order it was measured. Previously data was implicitly sorted by coordinates during the export.
+* When exporting Datasets to xarray where the shape is known, either by the use of utilities such as ``dond`` or manually specified, inferred parameters related to the setpoints and measured parameters are now included in the XArray Dataset.
+
+These changed were previously part of ``0.54.0`` but reverted in ``0.54.1`` since the change in coordinate sorting was not announced.

src/qcodes/dataset/exporters/export_to_xarray.py

@@ -83,10 +83,7 @@ def _load_to_xarray_dataset_dict_no_metadata(
             and sub_dict[name].shape == dataset.description.shapes[name]
         )
 
-        if shape_is_consistent and use_multi_index != "always" and False:
-            # disabled for now to think about how to best handle
-            # parameters that are measured in decreasing order
-            # where this export is inverted compared to the original export order
+        if shape_is_consistent and use_multi_index != "always":
             _LOG.info("Exporting %s to xarray using direct method", name)
             xr_dataset_dict[name] = _xarray_data_set_direct(dataset, name, sub_dict)
         else:

tests/dataset/measurement/test_inferred_parameters_fix.py

@@ -308,54 +308,53 @@ def test_inferred_parameters_in_actual_measurement_2d(
         )
 
     # xarray export
-    # disabled until we renable new export
-    # caplog.clear()
-    # with caplog.at_level("INFO"):
-    #     xarr = dataset.to_xarray_dataset()
-
-    # assert len(caplog.records) == 1
-    # if set_shape:
-    #     assert (
-    #         caplog.records[0].message
-    #         == "Exporting meas_parameter to xarray using direct method"
-    #     )
-    # else:
-    #     assert (
-    #         caplog.records[0].message
-    #         == "Exporting meas_parameter to xarray via pandas index"
-    #     )
-
-    # assert "meas_parameter" in xarr.data_vars
-
-    # assert "del_param_1" in xarr.coords
-    # assert xarr.coords["del_param_1"].shape == (num_points_x,)
-    # assert xarr.coords["del_param_1"].dims == ("del_param_1",)
-
-    # assert "del_param_2" in xarr.coords
-    # assert xarr.coords["del_param_2"].shape == (num_points_y,)
-    # assert xarr.coords["del_param_2"].dims == ("del_param_2",)
-
-    # if set_shape:
-    #     assert "dummy_dac_ch1" in xarr.coords
-    #     assert xarr.coords["dummy_dac_ch1"].shape == (num_points_x,)
-    #     assert xarr.coords["dummy_dac_ch1"].dims == ("del_param_1",)
-
-    #     assert "dummy_dac_ch2" in xarr.coords
-    #     assert xarr.coords["dummy_dac_ch2"].shape == (num_points_y,)
-    #     assert xarr.coords["dummy_dac_ch2"].dims == ("del_param_2",)
-    # else:
-    #     assert "dummy_dac_ch1" not in xarr.coords
-    #     assert "dummy_dac_ch2" not in xarr.coords
-    # assert xarr["meas_parameter"].dims == ("del_param_1", "del_param_2")
-    # assert xarr["meas_parameter"].shape == (num_points_x, num_points_y)
-
-    # # pandas export
-    # df = dataset.to_pandas_dataframe()
-    # assert "meas_parameter" in df.columns
-    # assert df.index.names == ["del_param_1", "del_param_2"]
-    # assert "dummy_dac_ch1" not in df.columns
-    # assert "dummy_dac_ch2" not in df.columns
-    # assert len(df) == total_points
+    caplog.clear()
+    with caplog.at_level("INFO"):
+        xarr = dataset.to_xarray_dataset()
+
+    assert len(caplog.records) == 1
+    if set_shape:
+        assert (
+            caplog.records[0].message
+            == "Exporting meas_parameter to xarray using direct method"
+        )
+    else:
+        assert (
+            caplog.records[0].message
+            == "Exporting meas_parameter to xarray via pandas index"
+        )
+
+    assert "meas_parameter" in xarr.data_vars
+
+    assert "del_param_1" in xarr.coords
+    assert xarr.coords["del_param_1"].shape == (num_points_x,)
+    assert xarr.coords["del_param_1"].dims == ("del_param_1",)
+
+    assert "del_param_2" in xarr.coords
+    assert xarr.coords["del_param_2"].shape == (num_points_y,)
+    assert xarr.coords["del_param_2"].dims == ("del_param_2",)
+
+    if set_shape:
+        assert "dummy_dac_ch1" in xarr.coords
+        assert xarr.coords["dummy_dac_ch1"].shape == (num_points_x,)
+        assert xarr.coords["dummy_dac_ch1"].dims == ("del_param_1",)
+
+        assert "dummy_dac_ch2" in xarr.coords
+        assert xarr.coords["dummy_dac_ch2"].shape == (num_points_y,)
+        assert xarr.coords["dummy_dac_ch2"].dims == ("del_param_2",)
+    else:
+        assert "dummy_dac_ch1" not in xarr.coords
+        assert "dummy_dac_ch2" not in xarr.coords
+    assert xarr["meas_parameter"].dims == ("del_param_1", "del_param_2")
+    assert xarr["meas_parameter"].shape == (num_points_x, num_points_y)
+
+    # pandas export
+    df = dataset.to_pandas_dataframe()
+    assert "meas_parameter" in df.columns
+    assert df.index.names == ["del_param_1", "del_param_2"]
+    assert "dummy_dac_ch1" not in df.columns
+    assert "dummy_dac_ch2" not in df.columns
+    assert len(df) == total_points
 
 
 def test_multiple_dependent_parameters_no_cross_contamination(

tests/dataset/test_dataset_export.py

@@ -1923,7 +1923,6 @@ def test_netcdf_export_with_mixed_timestamp_raw(
     assert loaded_ds.completed_timestamp_raw is None
 
 
-@pytest.mark.skip(reason="Disabled until we renable new export")
 @given(data=hst.data())
 @settings(
     max_examples=10,
@@ -2114,11 +2113,10 @@ def test_measurement_2d_with_inferred_setpoint(
     with caplog.at_level(logging.INFO):
         xr_ds = ds.to_xarray_dataset()
 
-    # disabled until we renable new export
-    # assert any(
-    #     "Exporting signal to xarray using direct method" in record.message
-    #     for record in caplog.records
-    # )
+    assert any(
+        "Exporting signal to xarray using direct method" in record.message
+        for record in caplog.records
+    )
 
     # Sizes and coords
     assert xr_ds.sizes == {"x": nx, "y": ny}
@@ -2130,16 +2128,15 @@ def test_measurement_2d_with_inferred_setpoint(
     expected_signal = x_vals[:, None] + 3.0 * y_vals[None, :]
     np.testing.assert_allclose(xr_ds["signal"].values, expected_signal)
 
-    # disabled until we renable new export
-    # # Inferred coords for y_b0 and y_b1 exist with dims only along y
-    # for name, vals in ("y_b0", y_b0_vals), ("y_b1", y_b1_vals):
-    #     assert name in xr_ds.coords
-    #     assert xr_ds.coords[name].dims == ("y",)
-    #     np.testing.assert_allclose(xr_ds.coords[name].values, vals)
-    #     # Indexes of inferred coords should correspond to the y axis index
-    #     inf_idx = xr_ds.coords[name].indexes
-    #     assert set(inf_idx.keys()) == {"y"}
-    #     assert inf_idx["y"].equals(xr_ds.indexes["y"])
+    # Inferred coords for y_b0 and y_b1 exist with dims only along y
+    for name, vals in ("y_b0", y_b0_vals), ("y_b1", y_b1_vals):
+        assert name in xr_ds.coords
+        assert xr_ds.coords[name].dims == ("y",)
+        np.testing.assert_allclose(xr_ds.coords[name].values, vals)
+        # Indexes of inferred coords should correspond to the y axis index
+        inf_idx = xr_ds.coords[name].indexes
+        assert set(inf_idx.keys()) == {"y"}
+        assert inf_idx["y"].equals(xr_ds.indexes["y"])
 
 
 def test_measurement_2d_with_inferred_setpoint_from_setpoint(
@@ -2181,11 +2178,10 @@ def test_measurement_2d_with_inferred_setpoint_from_setpoint(
     with caplog.at_level(logging.INFO):
         xr_ds = ds.to_xarray_dataset()
 
-    # disabled until we renable new export
-    # assert any(
-    #     "Exporting signal to xarray using direct method" in record.message
-    #     for record in caplog.records
-    # )
+    assert any(
+        "Exporting signal to xarray using direct method" in record.message
+        for record in caplog.records
+    )
 
     # Sizes and coords
     assert xr_ds.sizes == {"x": nx, "y": ny}
@@ -2240,24 +2236,22 @@ def test_measurement_2d_top_level_inferred_is_data_var(
         xr_ds = ds.to_xarray_dataset()
 
     # Direct path log should be present
-    # disabled until we renable new export
-    # assert any(
-    #     "Exporting signal to xarray using direct method" in record.message
-    #     for record in caplog.records
-    # )
+    assert any(
+        "Exporting signal to xarray using direct method" in record.message
+        for record in caplog.records
+    )
 
     # The derived param should be a data variable with dims (x, y), not a coord
-    # assert "derived" in xr_ds.data_vars
-    # assert "derived" not in xr_ds.coords
-    # assert xr_ds["derived"].dims == ("x", "y")
+    assert "derived" in xr_ds.data_vars
+    assert "derived" not in xr_ds.coords
+    assert xr_ds["derived"].dims == ("x", "y")
 
     expected_signal = x_vals[:, None] + y_vals[None, :]
-    # expected_derived = 2.0 * expected_signal
+    expected_derived = 2.0 * expected_signal
     np.testing.assert_allclose(xr_ds["signal"].values, expected_signal)
-    # np.testing.assert_allclose(xr_ds["derived"].values, expected_derived)
+    np.testing.assert_allclose(xr_ds["derived"].values, expected_derived)
 
 
-@pytest.mark.skip(reason="Disabled until we renable new export")
 def test_with_without_shape_is_the_same(experiment: Experiment) -> None:
     nx, ny = 2, 3
     x_vals = np.linspace(0.0, -1.0, nx)