Upgrade lsdb to >=0.8

tox.ini

@@ -36,7 +36,7 @@ deps =
     # Pandas functionality we use requires lxml, but it's not yet available as optional extras for 1.5.2
     oldestdeps: lxml
     # Oldest lsdb is not compatible with the versions above, we skip lsdb notebooks for oldestdeps job
-    # oldestdeps: lsdb==0.6.6
+    # oldestdeps: lsdb==0.8.1
 
     # Ugly workaround for the custom install_command to ensure the the arguments are properly passed into pip
     !oldestdeps: pip

tutorial_requirements.txt

@@ -12,7 +12,8 @@ pyarrow>=10.0.1
 hpgeom
 pandas[xml]>=1.5.2
 dask[distributed]
-lsdb>=0.6.6,<0.8
+# lsdb<0.8 returns wrong results for some catalogs, e.g. ZTF DR24 Lightcurves
+lsdb>=0.8.1
 psutil
 ray
 s3fs

tutorials/techniques-and-tools/irsa-hats-with-lsdb.md

@@ -56,7 +56,7 @@ We will use lsdb to leverage HATS partitioning for performing fast spatial queri
 
 ```{code-cell} ipython3
 # Uncomment the next line to install dependencies if needed.
-# !pip install s3fs "lsdb>=0.6.6,<0.8" pyarrow pandas numpy astropy dask matplotlib
+# !pip install s3fs "lsdb>=0.8.1" pyarrow pandas numpy astropy dask matplotlib
 ```
 
 ```{code-cell} ipython3
@@ -392,9 +392,10 @@ Since ZTF objects are defined per band, setting `n_neighbors=1` means this is on
 ```{code-cell} ipython3
 euclid_x_ztf = euclid_cone.crossmatch(
     ztf_cone,
-    suffixes=("_euclid", "_ztf"), # to distinguish columns from the two catalogs
     n_neighbors=1, # default is 1 too, can be tweaked
-    radius_arcsec=1  # default is 1 arcsec too, can be tweaked
+    radius_arcsec=1,  # default is 1 arcsec too, can be tweaked
+    suffixes=("_euclid", "_ztf"), # to distinguish columns from the two catalogs
+    suffix_method="all_columns",
 )
 euclid_x_ztf
 ```
@@ -562,10 +563,17 @@ ztf_lcs
 ```
 
 As earlier, this creates a lazy catalog object with the partition(s) that contains our IDs.
-We can load the light curves data into a DataFrame by using the `compute()` method:
+We can load the light curves data into a DataFrame by using the `compute()` method.
+Note: You may see a memory warning from lsdb which is expected due to the large size of Lightcurve data.
 
 ```{code-cell} ipython3
-ztf_lcs_df = ztf_lcs.compute() # ID search runs out of memory if we try to parallelize it with Dask client
+with Client(n_workers=get_nworkers(ztf_lcs),
+            threads_per_worker=1,
+            memory_limit=None # to prevent it from running out of memory
+            ) as client:
+    print(f"This may take more than a few minutes to complete. You can monitor progress in Dask dashboard at {client.dashboard_link}")
+    ztf_lcs_df = ztf_lcs.compute()
+
 ztf_lcs_df
 ```