IDs from STEREO

See following notebooks for details:

• Magnetic field data
• State data

STEREO magnetic field is already in RTN coordinates, so no need to transform it.

Download data using pyspedas, but load it using pycdfpp (using pyspedas to load the data directly into xarray is very slow)

Note STEREO-A plastic data file is “Non compliant ISTP file: No data variable found, this is suspicious”, -9.999999848243207e+30 instead of -1e+31 is used as fill value.

wget -r -np -nH -R "index.html*" https://spdf.gsfc.nasa.gov/pub/data/stereo/ahead/l2/plastic/1dmax/1min/201{1,2,3,4,5,6}/

import speasy as spz
from space_analysis.utils.speasy import Variables
from space_analysis.io.cdf import cdf2pl
from discontinuitypy.datasets import IDsDataset
from beforerr.polars import pl_norm
from discontinuitypy.utils.basic import resample

from datetime import timedelta
import polars as pl
import polars.selectors as cs

from sunpy.time import TimeRange
import pyspedas

from tqdm import tqdm

25-Feb-24 01:01:20: UserWarning: Traceback (most recent call last):
  File "/Users/zijin/micromamba/envs/psp_conjunction/lib/python3.11/site-packages/pdpipe/__init__.py", line 85, in <module>
    from . import skintegrate
  File "/Users/zijin/micromamba/envs/psp_conjunction/lib/python3.11/site-packages/pdpipe/skintegrate.py", line 20, in <module>
    from sklearn.base import BaseEstimator
ModuleNotFoundError: No module named 'sklearn'


25-Feb-24 01:01:20: UserWarning: pdpipe: Scikit-learn or skutil import failed. Scikit-learn-dependent pipeline stages will not be loaded.

25-Feb-24 01:01:20: UserWarning: Traceback (most recent call last):
  File "/Users/zijin/micromamba/envs/psp_conjunction/lib/python3.11/site-packages/pdpipe/__init__.py", line 105, in <module>
    from . import nltk_stages
  File "/Users/zijin/micromamba/envs/psp_conjunction/lib/python3.11/site-packages/pdpipe/nltk_stages.py", line 19, in <module>
    import nltk
ModuleNotFoundError: No module named 'nltk'


25-Feb-24 01:01:20: UserWarning: pdpipe: nltk import failed. nltk-dependent  pipeline stages will not be loaded.

import speasy as spz
entries = spz.core.cache.entries()
# drop internal entries
entries = list(filter(lambda e: '__internal__' not in e, entries))
print(entries[::-1])

['cda/WI_PLSP_3DP/MOM.P.MAGT3/2019-04-12T00:00:00+00:00', 'cda/WI_PLSP_3DP/MOM.P.MAGT3/2019-04-11T12:00:00+00:00', 'cda/WI_PLSP_3DP/MOM.P.MAGT3/2019-04-11T00:00:00+00:00', 'cda/WI_PLSP_3DP/MOM.P.MAGT3/2019-04-10T12:00:00+00:00', 'cda/WI_PLSP_3DP/MOM.P.MAGT3/2019-04-10T00:00:00+00:00', 'cda/WI_PLSP_3DP/MOM.P.MAGT3/2019-04-09T12:00:00+00:00', 'cda/WI_PLSP_3DP/MOM.P.MAGT3/2019-04-09T00:00:00+00:00', 'cda/WI_PLSP_3DP/MOM.P.MAGT3/2019-04-08T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/wsX7LELp/wi_plsps_3dp_20190408120000_20190412115954.cdf', 'https://cdaweb.gsfc.nasa.gov/tmp/wss6lQ5G/wi_plsps_3dp_20190408120000_20190412115954.cdf', 'https://cdaweb.gsfc.nasa.gov/tmp/wsgD6oUy/wi_plsps_3dp_20190408120000_20190412115954.cdf', 'https://cdaweb.gsfc.nasa.gov/tmp/ws2WbWGk/wi_plsps_3dp_20190408120000_20190412115954.cdf', 'https://cdaweb.gsfc.nasa.gov/tmp/wsw6URGt/wi_plsps_3dp_20190408120000_20190412115954.cdf', 'https://cdaweb.gsfc.nasa.gov/tmp/wsrsvJ7P/wi_plsps_3dp_20190408120000_20190412115954.cdf', 'cda/THB_L2_MOM/thb_peim_ptens_magQ/2019-04-12T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_ptens_magQ/2019-04-11T12:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_ptens_magQ/2019-04-11T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_ptens_magQ/2019-04-10T12:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_ptens_magQ/2019-04-10T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_ptens_magQ/2019-04-09T12:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_ptens_magQ/2019-04-09T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_ptens_magQ/2019-04-08T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/wsJysIZz/thb_l2s_mom_20190408120000_20190412115958.cdf', 'cda/THB_L2_MOM/thb_peem_ptens_magQ/2019-04-12T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peem_ptens_magQ/2019-04-11T12:00:00+00:00', 'cda/THB_L2_MOM/thb_peem_ptens_magQ/2019-04-11T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peem_ptens_magQ/2019-04-10T12:00:00+00:00', 'cda/THB_L2_MOM/thb_peem_ptens_magQ/2019-04-10T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peem_ptens_magQ/2019-04-09T12:00:00+00:00', 'cda/THB_L2_MOM/thb_peem_ptens_magQ/2019-04-09T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peem_ptens_magQ/2019-04-08T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/ws2UEYKB/thb_l2s_mom_20190408120000_20190412115958.cdf', 'cda/WI_PM_3DP/P_TEMP/2019-04-12T00:00:00+00:00', 'cda/WI_PM_3DP/P_TEMP/2019-04-11T12:00:00+00:00', 'cda/WI_PM_3DP/P_TEMP/2019-04-11T00:00:00+00:00', 'cda/WI_PM_3DP/P_TEMP/2019-04-10T12:00:00+00:00', 'cda/WI_PM_3DP/P_TEMP/2019-04-10T00:00:00+00:00', 'cda/WI_PM_3DP/P_TEMP/2019-04-09T12:00:00+00:00', 'cda/WI_PM_3DP/P_TEMP/2019-04-09T00:00:00+00:00', 'cda/WI_PM_3DP/P_TEMP/2019-04-08T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/wspZgXLc/wi_pms_3dp_20190408120000_20190412115958.cdf', 'cda/WI_PM_3DP/P_VELS/2019-04-12T00:00:00+00:00', 'cda/WI_PM_3DP/P_VELS/2019-04-11T12:00:00+00:00', 'cda/WI_PM_3DP/P_VELS/2019-04-11T00:00:00+00:00', 'cda/WI_PM_3DP/P_VELS/2019-04-10T12:00:00+00:00', 'cda/WI_PM_3DP/P_VELS/2019-04-10T00:00:00+00:00', 'cda/WI_PM_3DP/P_VELS/2019-04-09T12:00:00+00:00', 'cda/WI_PM_3DP/P_VELS/2019-04-09T00:00:00+00:00', 'cda/WI_PM_3DP/P_VELS/2019-04-08T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/wsgZ9Qeq/wi_pms_3dp_20190408120000_20190412115958.cdf', 'cda/WI_PM_3DP/P_DENS/2019-04-12T00:00:00+00:00', 'cda/WI_PM_3DP/P_DENS/2019-04-11T12:00:00+00:00', 'cda/WI_PM_3DP/P_DENS/2019-04-11T00:00:00+00:00', 'cda/WI_PM_3DP/P_DENS/2019-04-10T12:00:00+00:00', 'cda/WI_PM_3DP/P_DENS/2019-04-10T00:00:00+00:00', 'cda/WI_PM_3DP/P_DENS/2019-04-09T12:00:00+00:00', 'cda/WI_PM_3DP/P_DENS/2019-04-09T00:00:00+00:00', 'cda/WI_PM_3DP/P_DENS/2019-04-08T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/wstZl9Kd/wi_pms_3dp_20190408120000_20190412115958.cdf', 'cda/WI_H2_MFI/BGSE/2019-04-12T00:00:00+00:00', 'cda/WI_H2_MFI/BGSE/2019-04-11T12:00:00+00:00', 'cda/WI_H2_MFI/BGSE/2019-04-11T00:00:00+00:00', 'cda/WI_H2_MFI/BGSE/2019-04-10T12:00:00+00:00', 'cda/WI_H2_MFI/BGSE/2019-04-10T00:00:00+00:00', 'cda/WI_H2_MFI/BGSE/2019-04-09T12:00:00+00:00', 'cda/WI_H2_MFI/BGSE/2019-04-09T00:00:00+00:00', 'cda/WI_H2_MFI/BGSE/2019-04-08T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/wsxSYUNJ/wi_h2s_mfi_20190408120000_20190412115959.cdf', 'cda/THB_L2_MOM/thb_peim_ptotQ/2019-04-12T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_ptotQ/2019-04-11T12:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_ptotQ/2019-04-11T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_ptotQ/2019-04-10T12:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_ptotQ/2019-04-10T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_ptotQ/2019-04-09T12:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_ptotQ/2019-04-09T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_ptotQ/2019-04-08T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/wsFDFj77/thb_l2s_mom_20190408120000_20190412115958.cdf', 'cda/THB_L2_MOM/thb_peim_velocity_gseQ/2019-04-12T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_velocity_gseQ/2019-04-11T12:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_velocity_gseQ/2019-04-11T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_velocity_gseQ/2019-04-10T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/wsOxDZT7/thb_l2s_mom_20190410120000_20190412115958.cdf', 'cda/THB_L2_MOM/thb_peim_velocity_gseQ/2019-04-09T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_velocity_gseQ/2019-04-08T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/wsHyWZk9/thb_l2s_mom_20190408120000_20190409115958.cdf', 'cda/THB_L2_MOM/thb_peim_densityQ/2019-04-12T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_densityQ/2019-04-11T12:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_densityQ/2019-04-11T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_densityQ/2019-04-10T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/wsOkO4D6/thb_l2s_mom_20190410120000_20190412115958.cdf', 'cda/THB_L2_MOM/thb_peim_densityQ/2019-04-09T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_densityQ/2019-04-08T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/wsjw9VQW/thb_l2s_mom_20190408120000_20190409115958.cdf', 'cda/THB_L2_FGM/thb_fgl_gse/2019-04-12T00:00:00+00:00', 'cda/THB_L2_FGM/thb_fgl_gse/2019-04-11T12:00:00+00:00', 'cda/THB_L2_FGM/thb_fgl_gse/2019-04-11T00:00:00+00:00', 'cda/THB_L2_FGM/thb_fgl_gse/2019-04-10T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/wsPOK59Q/thb_l2s_fgm_20190410154948_20190412051612.cdf', 'cda/THB_L2_FGM/thb_fgl_gse/2019-04-09T00:00:00+00:00', 'cda/THB_L2_FGM/thb_fgl_gse/2019-04-08T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/wskTkvRu/thb_l2s_fgm_20190408120000_20190408192100.cdf', 'cda/THB_L2_MOM/thb_peim_densityQ/2019-04-10T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_densityQ/2019-04-09T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/wsjUpKuw/thb_l2s_mom_20190409120003_20190410115956.cdf', 'cda/THB_L2_FGM/thb_fgl_gse/2019-04-10T00:00:00+00:00', 'cda/THB_L2_FGM/thb_fgl_gse/2019-04-09T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/wsfWpYWe/thb_l2s_fgm_20190409122948_20190409223612.cdf', 'cda/THB_L2_MOM/thb_peim_velocity_gseQ/2019-04-10T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_velocity_gseQ/2019-04-09T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/wsF7wqKb/thb_l2s_mom_20190409120003_20190410115956.cdf', 'cda/SOLO_L2_MAG-RTN-NORMAL-1-MINUTE/B_RTN/2021-01-02T00:00:00+00:00', 'cda/SOLO_L2_MAG-RTN-NORMAL-1-MINUTE/B_RTN/2021-01-01T12:00:00+00:00', 'cda/SOLO_L2_MAG-RTN-NORMAL-1-MINUTE/B_RTN/2021-01-01T00:00:00+00:00', 'cda/SOLO_L2_MAG-RTN-NORMAL-1-MINUTE/B_RTN/2020-12-31T12:00:00+00:00', 'cda/WI_PLSP_3DP/ENERGY/2019-04-12T00:00:00+00:00', 'cda/WI_PLSP_3DP/ENERGY/2019-04-11T12:00:00+00:00', 'cda/WI_PLSP_3DP/ENERGY/2019-04-11T00:00:00+00:00', 'cda/WI_PLSP_3DP/ENERGY/2019-04-10T12:00:00+00:00', 'cda/WI_PLSP_3DP/ENERGY/2019-04-10T00:00:00+00:00', 'cda/WI_PLSP_3DP/ENERGY/2019-04-09T12:00:00+00:00', 'cda/WI_PLSP_3DP/ENERGY/2019-04-09T00:00:00+00:00', 'cda/WI_PLSP_3DP/ENERGY/2019-04-08T12:00:00+00:00', 'cda/WI_ELM2_3DP/MAGT3/2019-04-12T00:00:00+00:00', 'cda/WI_ELM2_3DP/MAGT3/2019-04-11T12:00:00+00:00', 'cda/WI_ELM2_3DP/MAGT3/2019-04-11T00:00:00+00:00', 'cda/WI_ELM2_3DP/MAGT3/2019-04-10T12:00:00+00:00', 'cda/WI_ELM2_3DP/MAGT3/2019-04-10T00:00:00+00:00', 'cda/WI_ELM2_3DP/MAGT3/2019-04-09T12:00:00+00:00', 'cda/WI_ELM2_3DP/MAGT3/2019-04-09T00:00:00+00:00', 'cda/WI_ELM2_3DP/MAGT3/2019-04-08T12:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_t3_magQ/2019-04-12T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_t3_magQ/2019-04-11T12:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_t3_magQ/2019-04-11T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_t3_magQ/2019-04-10T12:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_t3_magQ/2019-04-10T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_t3_magQ/2019-04-09T12:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_t3_magQ/2019-04-09T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peim_t3_magQ/2019-04-08T12:00:00+00:00', 'cda/THB_L2_MOM/thb_peem_t3_magQ/2019-04-12T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peem_t3_magQ/2019-04-11T12:00:00+00:00', 'cda/THB_L2_MOM/thb_peem_t3_magQ/2019-04-11T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peem_t3_magQ/2019-04-10T12:00:00+00:00', 'cda/THB_L2_MOM/thb_peem_t3_magQ/2019-04-10T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peem_t3_magQ/2019-04-09T12:00:00+00:00', 'cda/THB_L2_MOM/thb_peem_t3_magQ/2019-04-09T00:00:00+00:00', 'cda/THB_L2_MOM/thb_peem_t3_magQ/2019-04-08T12:00:00+00:00', 'cda/PSP_SWP_SPC_L3I/np_moment_gd/2019-04-07T12:00:00+00:00', 'cda/PSP_SWP_SPC_L3I/np_moment_gd/2019-04-07T00:00:00+00:00', 'cda/PSP_SWP_SPC_L3I/np_moment_gd/2019-04-06T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/wshv6OPm/psp_swps_spc_l3i_20190406120012_20190407235805.cdf', 'cda/PSP_SWP_SPI_SF00_L3_MOM/SUN_DIST/2019-04-07T12:00:00+00:00', 'cda/PSP_SWP_SPI_SF00_L3_MOM/SUN_DIST/2019-04-07T00:00:00+00:00', 'cda/PSP_SWP_SPI_SF00_L3_MOM/SUN_DIST/2019-04-06T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/wszcUkYV/psp_swps_spi_sf00_l3_mom_20190406120001_20190407235959.cdf', 'cda/PSP_SWP_SPI_SF00_L3_MOM/TEMP/2019-04-07T12:00:00+00:00', 'cda/PSP_SWP_SPI_SF00_L3_MOM/TEMP/2019-04-07T00:00:00+00:00', 'cda/PSP_SWP_SPI_SF00_L3_MOM/TEMP/2019-04-06T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/wsWQttvt/psp_swps_spi_sf00_l3_mom_20190406120001_20190407235959.cdf', 'cda/PSP_SWP_SPI_SF00_L3_MOM/VEL_RTN_SUN/2019-04-07T12:00:00+00:00', 'cda/PSP_SWP_SPI_SF00_L3_MOM/VEL_RTN_SUN/2019-04-07T00:00:00+00:00', 'cda/PSP_SWP_SPI_SF00_L3_MOM/VEL_RTN_SUN/2019-04-06T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/wsqD3Vl4/psp_swps_spi_sf00_l3_mom_20190406120001_20190407235959.cdf', 'cda/PSP_SWP_SPI_SF00_L3_MOM/DENS/2019-04-07T12:00:00+00:00', 'cda/PSP_SWP_SPI_SF00_L3_MOM/DENS/2019-04-07T00:00:00+00:00', 'cda/PSP_SWP_SPI_SF00_L3_MOM/DENS/2019-04-06T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/ws1kPspF/psp_swps_spi_sf00_l3_mom_20190406120001_20190407235959.cdf', 'cda/PSP_FLD_L3_RFS_LFR_QTN/N_elec/2019-04-07T12:00:00+00:00', 'cda/PSP_FLD_L3_RFS_LFR_QTN/N_elec/2019-04-07T00:00:00+00:00', 'cda/PSP_FLD_L3_RFS_LFR_QTN/N_elec/2019-04-06T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/wscwwzey/psp_flds_l3_rfs_lfr_qtn_20190406120006_20190407235922.cdf', 'cda/PSP_SWP_SPC_L3I/wp_moment_gd/2019-04-07T12:00:00+00:00', 'cda/PSP_SWP_SPC_L3I/wp_moment_gd/2019-04-07T00:00:00+00:00', 'cda/PSP_SWP_SPC_L3I/wp_moment_gd/2019-04-06T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/wsqkNkGL/psp_swps_spc_l3i_20190406120012_20190407235805.cdf', 'cda/PSP_SWP_SPC_L3I/vp_moment_RTN_gd/2019-04-07T12:00:00+00:00', 'cda/PSP_SWP_SPC_L3I/vp_moment_RTN_gd/2019-04-07T00:00:00+00:00', 'cda/PSP_SWP_SPC_L3I/vp_moment_RTN_gd/2019-04-06T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/wsBcRYzd/psp_swps_spc_l3i_20190406120012_20190407235805.cdf', 'cda/PSP_FLD_L3_SQTN_RFS_V1V2/electron_density/2019-04-07T12:00:00+00:00', 'cda/PSP_FLD_L3_SQTN_RFS_V1V2/electron_density/2019-04-07T00:00:00+00:00', 'cda/PSP_FLD_L3_SQTN_RFS_V1V2/electron_density/2019-04-06T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/ws5ziTed/psp_flds_l3_sqtn_rfs_v1v2_20190406120006_20190407235957.cdf', 'cda/PSP_FLD_L2_MAG_RTN/psp_fld_l2_mag_RTN/2019-04-07T12:00:00+00:00', 'cda/PSP_FLD_L2_MAG_RTN/psp_fld_l2_mag_RTN/2019-04-07T00:00:00+00:00', 'cda/PSP_FLD_L2_MAG_RTN/psp_fld_l2_mag_RTN/2019-04-06T12:00:00+00:00', 'https://cdaweb.gsfc.nasa.gov/tmp/wsZx9P7S/psp_flds_l2_mag_rtn_20190406120000_20190407235735.cdf', 'https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/thb_l2_mom_00000000_v01.cdf', 'https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/thb_l2_fgm_00000000_v01.cdf', 'https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/sta_l2_pla_1dmax_1min_00000000_v01.cdf', 'https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/sta_l2_magplasma_1m_00000000_v01.cdf', 'https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/sta_l1_mag_rtn_00000000_v01.cdf', 'https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/wi_k0_swe_00000000_v01.cdf', 'https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/wi_h4-rtn_mfi_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/the/l2/sst/0000/the_l2_sst_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/the/l2/scm/0000/the_l2_scm_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/the/l2/mom/0000/the_l2_mom_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/the/l2/gmom/0000/the_l2_gmom_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/the/l2/fit/0000/the_l2_fit_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/the/l2/fgm/0000/the_l2_fgm_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/the/l2/fft/0000/the_l2_fft_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/the/l2/fbk/0000/the_l2_fbk_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/the/l2/esd/0000/the_l2_esd_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/the/l2/esa/0000/the_l2_esa_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/the/l2/efi/0000/the_l2_efi_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thd/l2/sst/0000/thd_l2_sst_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thd/l2/scm/0000/thd_l2_scm_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thd/l2/mom/0000/thd_l2_mom_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thd/l2/gmom/0000/thd_l2_gmom_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thd/l2/fit/0000/thd_l2_fit_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thd/l2/fgm/0000/thd_l2_fgm_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thd/l2/fft/0000/thd_l2_fft_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thd/l2/fbk/0000/thd_l2_fbk_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thd/l2/esd/0000/thd_l2_esd_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thd/l2/esa/0000/thd_l2_esa_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thd/l2/efi/0000/thd_l2_efi_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thc/l2/sst/0000/thc_l2_sst_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thc/l2/scm/0000/thc_l2_scm_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thc/l2/mom/0000/thc_l2_mom_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thc/l2/gmom/0000/thc_l2_gmom_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thc/l2/fit/0000/thc_l2_fit_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thc/l2/fgm/0000/thc_l2_fgm_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thc/l2/fft/0000/thc_l2_fft_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thc/l2/fbk/0000/thc_l2_fbk_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thc/l2/esd/0000/thc_l2_esd_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thc/l2/esa/0000/thc_l2_esa_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thc/l2/efi/0000/thc_l2_efi_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thb/l2/sst/0000/thb_l2_sst_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thb/l2/scm/0000/thb_l2_scm_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thb/l2/mom/0000/thb_l2_mom_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thb/l2/gmom/0000/thb_l2_gmom_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thb/l2/fit/0000/thb_l2_fit_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thb/l2/fgm/0000/thb_l2_fgm_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thb/l2/fft/0000/thb_l2_fft_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thb/l2/fbk/0000/thb_l2_fbk_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thb/l2/esd/0000/thb_l2_esd_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thb/l2/esa/0000/thb_l2_esa_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/thb/l2/efi/0000/thb_l2_efi_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/tha/l2/sst/0000/tha_l2_sst_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/tha/l2/scm/0000/tha_l2_scm_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/tha/l2/mom/0000/tha_l2_mom_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/tha/l2/gmom/0000/tha_l2_gmom_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/tha/l2/fit/0000/tha_l2_fit_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/tha/l2/fgm/0000/tha_l2_fgm_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/tha/l2/fft/0000/tha_l2_fft_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/tha/l2/fbk/0000/tha_l2_fbk_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/tha/l2/esd/0000/tha_l2_esd_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/tha/l2/esa/0000/tha_l2_esa_00000000_v01.cdf', 'http://cdpp.irap.omp.eu/themisdata/tha/l2/efi/0000/tha_l2_efi_00000000_v01.cdf', 'https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms4_fpi_fast_l2_des-moms_00000000_v01.cdf', 'https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms4_fpi_brst_l2_des-moms_00000000_v01.cdf', 'https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms4_fgm_srvy_l2_00000000_v01.cdf', 'https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms3_fpi_fast_l2_des-moms_00000000_v01.cdf', 'https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms3_fpi_brst_l2_des-moms_00000000_v01.cdf', 'https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms3_fgm_srvy_l2_00000000_v01.cdf', 'https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms2_fpi_fast_l2_des-moms_00000000_v01.cdf', 'https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms2_fpi_brst_l2_des-moms_00000000_v01.cdf', 'https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms2_fgm_srvy_l2_00000000_v01.cdf', 'https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms1_fpi_fast_l2_des-moms_00000000_v01.cdf', 'https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms1_fpi_brst_l2_des-moms_00000000_v01.cdf', 'https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms1_fgm_srvy_l2_00000000_v01.cdf', 'https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/erg_pwe_hfa_l3_1min_00000000_v01.cdf', 'https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/erg_lepe_l3_pa_00000000_v01.cdf', 'cache/version']

spz.update_inventories()

timerange = ["2011-08-25", "2016-06-30"]
# timerange = ["2011-08-25", "2011-09-01"]
mission = "STA"
ts = timedelta(seconds=1)
tau = timedelta(seconds=60)

provider = 'archive/local'
mag_dataset = "STA_L1_MAG_RTN"
mag_parameters = ["BFIELD"]

# plasma_dataset = "STA_L2_MAGPLASMA_1M"
# plasma_parameters= ["Np", "Cone_Angle", "Vt_Over_V_RTN", "Vp_RTN", "Tp"]

plasma_dataset = 'STA_L2_PLA_1DMAX_1MIN'
density_col = 'proton_number_density'
vec_cols = ['proton_Vr_RTN', 'proton_Vt_RTN', 'proton_Vn_RTN']
temperature_col = 'proton_temperature'
plasma_parameters = [density_col] + vec_cols + [temperature_col]

fmt = 'arrow'
fname = f"../../../data/05_reporting/events.{mission}.ts_{ts.total_seconds():.2f}s_tau_60s.{fmt}"

Downloading files

def download(timerange):
    
    files = pyspedas.stereo.plastic(timerange, downloadonly=True)

    for file in files:
        parquet_file = file.replace('.cdf', f'.{fmt}')
        cdf2pl(file, plasma_parameters).collect().write_ipc(parquet_file)
        
    mag_files = pyspedas.stereo.mag(timerange, downloadonly=True)
    
    return files
    
        
# download(timerange)

Loading data

mag_vars = Variables(
    provider = provider,
    dataset=mag_dataset,
    parameters=mag_parameters,
    timerange=timerange,
).retrieve_data()

---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
Cell In[7], line 6
      1 mag_vars = Variables(
      2     provider = provider,
      3     dataset=mag_dataset,
      4     parameters=mag_parameters,
      5     timerange=timerange,
----> 6 ).retrieve_data()

File ~/micromamba/envs/psp_conjunction/lib/python3.11/site-packages/space_analysis/utils/speasy.py:86, in Variables.retrieve_data(self)
     83 def retrieve_data(self):
     84     # return Variables with data set
     85     if "local" in self.provider:
---> 86         self._data = spz.get_data(self.products, self.timerange)
     87     else:
     88         self._data = spz.get_data(
     89             self.products, self.timerange, disable_proxy=self._disable_proxy
     90         )

File ~/micromamba/envs/psp_conjunction/lib/python3.11/site-packages/speasy/core/requests_scheduling/request_dispatch.py:332, in get_data(*args, **kwargs)
    330 product = args[0]
    331 if is_collection(product) and not isinstance(product, SpeasyIndex):
--> 332     return list(map(lambda p: get_data(p, *args[1:], **kwargs), progress_bar(leave=True, **kwargs)(product)))
    334 if len(args) == 1:
    335     return _get_catalog_or_timetable(*args, **kwargs)

File ~/micromamba/envs/psp_conjunction/lib/python3.11/site-packages/speasy/core/requests_scheduling/request_dispatch.py:332, in get_data.<locals>.<lambda>(p)
    330 product = args[0]
    331 if is_collection(product) and not isinstance(product, SpeasyIndex):
--> 332     return list(map(lambda p: get_data(p, *args[1:], **kwargs), progress_bar(leave=True, **kwargs)(product)))
    334 if len(args) == 1:
    335     return _get_catalog_or_timetable(*args, **kwargs)

File ~/micromamba/envs/psp_conjunction/lib/python3.11/site-packages/speasy/core/requests_scheduling/request_dispatch.py:339, in get_data(*args, **kwargs)
    337 t_range = args[1]
    338 if _is_dtrange(t_range):
--> 339     return _get_timeserie1(*args, **kwargs)
    340 if is_collection(t_range):
    341     return list(
    342         map(lambda r: get_data(product, r, *args[2:], **kwargs),
    343             progress_bar(leave=False, **kwargs)(t_range)))

File ~/micromamba/envs/psp_conjunction/lib/python3.11/site-packages/speasy/core/requests_scheduling/request_dispatch.py:182, in _get_timeserie1(index, dtrange, **kwargs)
    181 def _get_timeserie1(index, dtrange, **kwargs):
--> 182     return _scalar_get_data(index, dtrange[0], dtrange[1], **kwargs)

File ~/micromamba/envs/psp_conjunction/lib/python3.11/site-packages/speasy/core/requests_scheduling/request_dispatch.py:173, in _scalar_get_data(index, *args, **kwargs)
    171 provider_uid, product_uid = provider_and_product(index)
    172 if provider_uid in PROVIDERS:
--> 173     return PROVIDERS[provider_uid].get_data(product_uid, *args, **kwargs)
    174 raise ValueError(f"Can't find a provider for {index}")

File ~/micromamba/envs/psp_conjunction/lib/python3.11/site-packages/speasy/core/__init__.py:223, in AllowedKwargs.__call__.<locals>.wrapped(*args, **kwargs)
    220 unexpected_args = list(
    221     filter(lambda arg_name: arg_name not in self.allowed_list, kwargs.keys()))
    222 if not unexpected_args:
--> 223     return func(*args, **kwargs)
    224 raise TypeError(
    225     f"Unexpected keyword argument {unexpected_args}, allowed keyword arguments are {self.allowed_list}")

File ~/micromamba/envs/psp_conjunction/lib/python3.11/site-packages/speasy/webservices/generic_archive/__init__.py:106, in GenericArchive.get_data(self, product, start_time, stop_time, **kwargs)
    103 @AllowedKwargs(GET_DATA_ALLOWED_KWARGS + CACHE_ALLOWED_KWARGS + ['force_refresh'])
    104 def get_data(self, product: str or ParameterIndex, start_time: AnyDateTimeType, stop_time: AnyDateTimeType,
    105              **kwargs) -> Optional[SpeasyVariable]:
--> 106     var = self._get_data(product=self._parameter_index(product), start_time=start_time, stop_time=stop_time,
    107                          **kwargs)
    108     return var

File ~/micromamba/envs/psp_conjunction/lib/python3.11/site-packages/speasy/webservices/generic_archive/__init__.py:114, in GenericArchive._get_data(self, product, start_time, stop_time, **kwargs)
    110 def _get_data(self, product: ParameterIndex, start_time: AnyDateTimeType, stop_time: AnyDateTimeType, **kwargs) -> \
    111 Optional[
    112     SpeasyVariable]:
    113     ga_cfg: dict = getattr(product, 'spz_ga_cfg')
--> 114     return get_product(**ga_cfg,
    115                        variable=product.spz_name(), start_time=start_time, stop_time=stop_time, **kwargs)

File ~/micromamba/envs/psp_conjunction/lib/python3.11/site-packages/speasy/core/direct_archive_downloader/direct_archive_downloader.py:162, in get_product(url_pattern, split_rule, variable, start_time, stop_time, use_file_list, **kwargs)
    159 def get_product(url_pattern: str, split_rule: str, variable: str, start_time: AnyDateTimeType,
    160                 stop_time: AnyDateTimeType, use_file_list: bool = False, **kwargs) -> Optional[SpeasyVariable]:
    161     if split_rule.lower() == "regular":
--> 162         return RegularSplitDirectDownload.get_product(url_pattern, variable, start_time, stop_time,
    163                                                       use_file_list, **kwargs)
    164     if split_rule.lower() == "random":
    165         return RandomSplitDirectDownload.get_product(url_pattern, variable, start_time, stop_time,
    166                                                      **kwargs)

File ~/micromamba/envs/psp_conjunction/lib/python3.11/site-packages/speasy/core/direct_archive_downloader/direct_archive_downloader.py:150, in RegularSplitDirectDownload.get_product(url_pattern, variable, start_time, stop_time, use_file_list, split_frequency, **kwargs)
    144 @staticmethod
    145 def get_product(url_pattern: str, variable: str, start_time: AnyDateTimeType,
    146                 stop_time: AnyDateTimeType, use_file_list: bool = False, split_frequency: str = "daily",
    147                 **kwargs) -> \
    148     Optional[SpeasyVariable]:
    149     v = merge(
--> 150         list(map(lambda date: _read_cdf(_build_url(url_pattern, date, use_file_list=use_file_list),
    151                                         variable=variable, **kwargs),
    152                  spilt_range(split_frequency=split_frequency, start_time=start_time,
    153                              stop_time=stop_time))))
    154     if v is not None:
    155         return v[make_utc_datetime(start_time):make_utc_datetime(stop_time)]

File ~/micromamba/envs/psp_conjunction/lib/python3.11/site-packages/speasy/core/direct_archive_downloader/direct_archive_downloader.py:150, in RegularSplitDirectDownload.get_product.<locals>.<lambda>(date)
    144 @staticmethod
    145 def get_product(url_pattern: str, variable: str, start_time: AnyDateTimeType,
    146                 stop_time: AnyDateTimeType, use_file_list: bool = False, split_frequency: str = "daily",
    147                 **kwargs) -> \
    148     Optional[SpeasyVariable]:
    149     v = merge(
--> 150         list(map(lambda date: _read_cdf(_build_url(url_pattern, date, use_file_list=use_file_list),
    151                                         variable=variable, **kwargs),
    152                  spilt_range(split_frequency=split_frequency, start_time=start_time,
    153                              stop_time=stop_time))))
    154     if v is not None:
    155         return v[make_utc_datetime(start_time):make_utc_datetime(stop_time)]

File ~/micromamba/envs/psp_conjunction/lib/python3.11/site-packages/speasy/core/direct_archive_downloader/direct_archive_downloader.py:26, in _read_cdf(url, variable, **kwargs)
     24     return None
     25 if is_local_file(url):
---> 26     return _local_read_cdf(file=url, variable=variable, **kwargs)
     27 return _remote_read_cdf(url=url, variable=variable, **kwargs)

File ~/micromamba/envs/psp_conjunction/lib/python3.11/site-packages/speasy/core/direct_archive_downloader/direct_archive_downloader.py:31, in _local_read_cdf(file, variable, **kwargs)
     30 def _local_read_cdf(file: str, variable: str, **kwargs) -> Optional[SpeasyVariable]:
---> 31     return load_variable(file=file, variable=variable)

File ~/micromamba/envs/psp_conjunction/lib/python3.11/site-packages/speasy/core/cdf/__init__.py:77, in load_variable(variable, file, cache_remote_files)
     75 if type(file) is str:
     76     if is_local_file(file):
---> 77         return _load_variable(variable=variable, file=urlparse(url=file).path)
     78     return _load_variable(variable=variable,
     79                           buffer=any_loc_open(file, mode='rb', cache_remote_files=cache_remote_files).read())
     80 if type(file) is bytes:

File ~/micromamba/envs/psp_conjunction/lib/python3.11/site-packages/speasy/core/cdf/__init__.py:52, in _load_variable(variable, file, buffer)
     50 if istp is not None:
     51     if variable in istp.data_variables():
---> 52         var = istp.data_variable(variable)
     53     elif variable.replace('-', '_') in istp.data_variables():  # THX CSA/ISTP
     54         var = istp.data_variable(variable.replace('-', '_'))

File ~/micromamba/envs/psp_conjunction/lib/python3.11/site-packages/pyistp/loader.py:21, in ISTPLoader.data_variable(self, var_name)
     20 def data_variable(self, var_name) -> DataVariable:
---> 21     return self._impl.data_variable(var_name)

File ~/micromamba/envs/psp_conjunction/lib/python3.11/site-packages/pyistp/_impl.py:114, in ISTPLoaderImpl.data_variable(self, var_name)
    113 def data_variable(self, var_name) -> DataVariable:
--> 114     return _load_data_var(self.cdf, var_name)

File ~/micromamba/envs/psp_conjunction/lib/python3.11/site-packages/pyistp/_impl.py:72, in _load_data_var(cdf, var)
     71 def _load_data_var(cdf: object, var: str) -> DataVariable or None:
---> 72     values = cdf.values(var)
     73     axes = _get_axes(cdf, var, values.shape)
     74     attributes = _get_attributes(cdf, var)

File ~/micromamba/envs/psp_conjunction/lib/python3.11/site-packages/pyistp/drivers/pycdfpp.py:56, in Driver.values(self, var, is_metadata_variable)
     54 if is_metadata_variable and self.is_char(var):
     55     return _drop_first_dim_if_nrv(v.is_nrv, v.values_encoded)
---> 56 return _drop_first_dim_if_nrv(v.is_nrv, v.values)

File ~/micromamba/envs/psp_conjunction/lib/python3.11/site-packages/pyistp/drivers/pycdfpp.py:5, in _drop_first_dim_if_nrv(is_nrv, values)
      1 import pycdfpp
      2 import numpy as np
----> 5 def _drop_first_dim_if_nrv(is_nrv: bool, values):
      6     if is_nrv:
      7         if values.shape[0] == 1:

KeyboardInterrupt: 

def get_and_process_data(timerange):
    mag_vars = Variables(
        provider = provider,
        dataset=mag_dataset,
        parameters=mag_parameters,
        timerange=timerange,
    ).retrieve_data()

    bcols = mag_vars.data[0].columns[:3]

    mag_data = mag_vars.to_polars().drop('BTotal')
    
    files = pyspedas.stereo.plastic(timerange, downloadonly=True, no_update=True)
    parquet_files = [file.replace('.cdf', f'.{fmt}') for file in files]

    plasma_data = pl.scan_ipc(parquet_files).with_columns(
        plasma_density = pl.when(pl.col(density_col) <0 ).then(None).otherwise(pl.col(density_col)),
        plasma_speed=pl_norm(vec_cols), ).rename({temperature_col:'plasma_temperature'}
    )
    
    return IDsDataset(
        mag_data=mag_data.pipe(resample, every=ts),
        plasma_data=plasma_data,
        tau=tau,
        ts=ts,
        bcols=bcols,
        vec_cols=vec_cols,
        density_col="plasma_density",
        speed_col="plasma_speed",
        temperature_col='plasma_temperature',
    ).find_events(return_best_fit=False).update_candidates_with_plasma_data()

ids : list[pl.DataFrame] = []
for _tr in tqdm(TimeRange(timerange).split(8)):
    
    _timerange = [_tr.start.value, _tr.end.value]
    _id = get_and_process_data(_timerange)
    ids.append(_id.events)
    
id = pl.concat(ids, how='vertical_relaxed').select(cs.datetime(), cs.duration(), cs.numeric())
id.write_ipc(fname)