Merge pull request #107 from sagasurvey/add-hipass-manual

Add hipass manually and refactor base prep function
sagasurvey · Jan 12, 2024 · e3db693 · e3db693
2 parents 87c517f + a79996a
commit e3db693
Show file tree

Hide file tree

Showing 5 changed files with 84 additions and 59 deletions.
diff --git a/SAGA/objects/build2.py b/SAGA/objects/build2.py
@@ -743,7 +743,7 @@ def get_tel_rank(
         has_HI = mask_within_dz & (specs_to_merge["HI_FLUX"] > 0)
         if has_HI.any():
             has_HI = np.argmax(has_HI)
-            for c in ["HI_FLUX", "HI_FLUX_ERR"]:
+            for c in ["HI_FLUX", "HI_FLUX_ERR", "HI_SOURCE"]:
                 specs[c][best_spec_index] = specs_to_merge[c][has_HI]
 
         nsa_specs = specs_to_merge[specs_to_merge["TELNAME"] == "NSA"]

diff --git a/SAGA/objects/build3.py b/SAGA/objects/build3.py
@@ -83,6 +83,74 @@ def prepare_objid(catalog):
     return release_short * np.int64(1e16) + catalog["BRICKID"] * np.int64(1e10) + catalog["OBJID"].astype(np.int64)
 
 
+def update_star_galaxy_flag(catalog):
+    sigma_grz = [f"SIGMA_GOOD_{b}" for b in "GRZ"]
+    fracflux_grz = [f"FRACFLUX_{b}" for b in "GRZ"]
+
+    bright_stars = Query(
+        "SHAPE_R < 1",
+        "r_mag < 17",
+        (Query("abs(PMRA * sqrt(PMRA_IVAR)) >= 2") | Query("abs(PMDEC * sqrt(PMDEC_IVAR)) >= 2")),
+    )
+
+    type_queries = {
+        0: bright_stars,
+        1: QueryMaker.equal("TYPE", "PSF"),
+        2: QueryMaker.equal("TYPE", "REX"),
+        3: QueryMaker.equal("TYPE", "EXP"),
+        4: QueryMaker.equal("TYPE", "DEV"),
+        5: QueryMaker.equal("TYPE", "SER"),
+        6: QueryMaker.equal("REF_CAT", "L3"),
+        7: Query("FITBITS % 2 > 0", "(FITBITS >> 5) % 16 == 0"),
+        8: reduce_compare(fracflux_grz, np.max, np.less, 0.7),
+        9: reduce_compare(sigma_grz, np.max, np.greater, 100),
+    }
+
+    catalog["morphology_info"] = get_remove_flag(catalog, type_queries)
+    catalog["is_galaxy"] = (
+        Query("morphology_info % 4 == 0")
+        | Query("(morphology_info >> 6) % 2 == 1")
+        | Query("(morphology_info >> 7) % 8 == 7")
+    ).mask(catalog)
+
+    return catalog
+
+
+def set_remove_flag(catalog, to_remove=None, to_recover=None, trim=True):
+    allmask_grz = [f"ALLMASK_{b}" for b in "GRZ"]
+    sigma_grz = [f"SIGMA_GOOD_{b}" for b in "GRZ"]
+    sigma_wise = [f"SIGMA_GOOD_W{b}" for b in "1234"]
+    fracflux_grz = [f"FRACFLUX_{b}" for b in "GRZ"]
+
+    remove_queries = {
+        1: reduce_compare(allmask_grz, np.median, np.greater, 0),
+        2: Query("FLUX_R <= 0"),
+        3: Query("FLUX_G <= 0", "FLUX_Z <= 0"),
+        4: (reduce_compare(sigma_grz, np.max, np.less, 60) & reduce_compare(fracflux_grz, np.max, np.greater, 2)),
+        5: reduce_compare(sigma_grz + sigma_wise, np.max, np.less, 20),
+        6: Query("(morphology_info >> 7) % 2 == 1", "(morphology_info >> 8) % 4 < 3"),
+    }
+
+    catalog["REMOVE"] = np.where(np.char.isalnum(catalog["REF_CAT"]), 0, get_remove_flag(catalog, remove_queries))
+    catalog["REF_CAT"] = np.char.strip(catalog["REF_CAT"])
+    catalog["SGA_ID"] = np.where(QueryMaker.equal("REF_CAT", "L3").mask(catalog), catalog["REF_ID"], -1)
+
+    if trim:
+        catalog = catalog[MERGED_CATALOG_COLUMNS]
+    catalog["survey"] = "decals"
+    catalog["OBJID_decals"] = catalog["OBJID"]
+    catalog["REMOVE_decals"] = catalog["REMOVE"]
+
+    if to_remove is not None:
+        catalog["REMOVE"] |= np.isin(catalog["OBJID"], to_remove).astype(np.int32)
+
+    if to_recover is not None:
+        idx = np.flatnonzero(np.isin(catalog["OBJID"], to_recover))
+        catalog["REMOVE"][idx] = 0
+
+    return catalog
+
+
 def prepare_decals_catalog_for_merging(catalog, to_remove=None, to_recover=None, trim=True):
     """
     Refs:
@@ -163,63 +231,8 @@ def prepare_decals_catalog_for_merging(catalog, to_remove=None, to_recover=None,
         catalog["r_mag"],
     )
 
-    allmask_grz = [f"ALLMASK_{b}" for b in "GRZ"]
-    sigma_grz = [f"SIGMA_GOOD_{b}" for b in "GRZ"]
-    sigma_wise = [f"SIGMA_GOOD_W{b}" for b in "1234"]
-    fracflux_grz = [f"FRACFLUX_{b}" for b in "GRZ"]
-
-    # Do galaxy/star separation
-    bright_stars = Query(
-        "SHAPE_R < 1",
-        "r_mag < 17",
-        (Query("abs(PMRA * sqrt(PMRA_IVAR)) >= 2") | Query("abs(PMDEC * sqrt(PMDEC_IVAR)) >= 2")),
-    )
-
-    type_queries = {
-        0: bright_stars,
-        1: QueryMaker.equal("TYPE", "PSF"),
-        2: QueryMaker.equal("TYPE", "REX"),
-        3: QueryMaker.equal("TYPE", "EXP"),
-        4: QueryMaker.equal("TYPE", "DEV"),
-        5: QueryMaker.equal("TYPE", "SER"),
-        6: QueryMaker.equal("REF_CAT", "L3"),
-        7: Query("FITBITS % 2 > 0", "(FITBITS >> 5) % 16 == 0"),
-        8: reduce_compare(fracflux_grz, np.max, np.less, 0.7),
-        9: reduce_compare(sigma_grz, np.max, np.greater, 100),
-    }
-
-    catalog["morphology_info"] = get_remove_flag(catalog, type_queries)
-    catalog["is_galaxy"] = (
-        Query("morphology_info % 4 == 0")
-        | Query("(morphology_info >> 6) % 2 == 1")
-        | Query("(morphology_info >> 7) % 8 == 7")
-    ).mask(catalog)
-
-    remove_queries = {
-        1: reduce_compare(allmask_grz, np.median, np.greater, 0),
-        2: Query("FLUX_R <= 0"),
-        3: Query("FLUX_G <= 0", "FLUX_Z <= 0"),
-        4: (reduce_compare(sigma_grz, np.max, np.less, 60) & reduce_compare(fracflux_grz, np.max, np.greater, 2)),
-        5: reduce_compare(sigma_grz + sigma_wise, np.max, np.less, 20),
-        6: Query("(morphology_info >> 7) % 2 == 1", "(morphology_info >> 8) % 4 < 3"),
-    }
-
-    catalog["REMOVE"] = np.where(np.char.isalnum(catalog["REF_CAT"]), 0, get_remove_flag(catalog, remove_queries))
-    catalog["REF_CAT"] = np.char.strip(catalog["REF_CAT"])
-    catalog["SGA_ID"] = np.where(QueryMaker.equal("REF_CAT", "L3").mask(catalog), catalog["REF_ID"], -1)
-
-    if trim:
-        catalog = catalog[MERGED_CATALOG_COLUMNS]
-    catalog["survey"] = "decals"
-    catalog["OBJID_decals"] = catalog["OBJID"]
-    catalog["REMOVE_decals"] = catalog["REMOVE"]
-
-    if to_remove is not None:
-        catalog["REMOVE"] |= np.isin(catalog["OBJID"], to_remove).astype(np.int32)
-
-    if to_recover is not None:
-        idx = np.flatnonzero(np.isin(catalog["OBJID"], to_recover))
-        catalog["REMOVE"][idx] = 0
+    catalog = update_star_galaxy_flag(catalog)
+    catalog = set_remove_flag(catalog, to_remove, to_recover, trim)
 
     return catalog
 
@@ -456,6 +469,12 @@ def identify_host(base):
     return base
 
 
+def apply_specs_manual_fixes(base):
+    fill_values_by_query(base, Query("OBJID == 902422980000000214"), {"HI_FLUX": 7.1, "HI_FLUX_ERR": 0.0178885438, "HI_SOURCE": "HIPASS"})
+    fill_values_by_query(base, Query("OBJID == 902799740000002207"), {"HI_FLUX": 16.9, "HI_FLUX_ERR": 0.0178885438, "HI_SOURCE": "HIPASS"})
+    return base
+
+
 def add_spec_phot_sep(base):
     has_any_spec_mask = base["ZQUALITY"] > -1
 
@@ -700,6 +719,7 @@ def build_full_stack(
         base = add_sga_specs(base, sga)
         del sga
 
+    base = apply_specs_manual_fixes(base)
     base = build2.remove_shreds_near_spec_obj(base)
 
     if halpha is not None:

diff --git a/SAGA/spectra/common.py b/SAGA/spectra/common.py
@@ -17,6 +17,7 @@
     ("HELIO_CORR", "|b1"),
     ("HI_FLUX", "<f4"),
     ("HI_FLUX_ERR", "<f4"),
+    ("HI_SOURCE", "<U8"),
 )
 
 SPECS_COLUMNS = dict(_SPECS_COLUMNS)
@@ -42,6 +43,8 @@ def ensure_specs_dtype(spectra, cols_definition=_SPECS_COLUMNS, skip_missing_col
                 raise ValueError("unknown spec type!")
         else:
             cols.append(c)
+        if hasattr(spectra[c], "unit") and hasattr(spectra[c], "value"):
+            spectra.replace_column(c, spectra[c].value)
         if spectra[c].dtype.str != t:
             spectra.replace_column(c, spectra[c].astype(t))
         if spectra[c].description:

diff --git a/SAGA/spectra/read_external.py b/SAGA/spectra/read_external.py
@@ -319,6 +319,7 @@ def read_alfalfa(file_path):
     specs["SPEC_Z_ERR"] = specs["sigW"].astype(np.float64) * 0.5 / SPEED_OF_LIGHT
     specs["ZQUALITY"] = np.where(valid_oc_coord, 3, 2)
 
+    specs["HI_SOURCE"] = "ALFALFA"
     specs["TELNAME"] = "ALFALF"
     specs["MASKNAME"] = "ALFALFA"
     specs["HELIO_CORR"] = True
@@ -346,6 +347,7 @@ def read_fashi(file_path):
     specs["ZQUALITY"] = np.where(valid_oc_coord, 3, 2)
     specs["HI_FLUX"] *= 1.0e-3  # convret from mJy to Jy
     specs["HI_FLUX_ERR"] *= 1.0e-3  # convret from mJy to Jy
+    specs["HI_SOURCE"] = "FASHI"
 
     specs["TELNAME"] = "FASHI"
     specs["MASKNAME"] = "FASHI"

diff --git a/SAGA/version.py b/SAGA/version.py
@@ -1,4 +1,4 @@
 """
 SAGA package version
 """
-__version__ = "0.69.0"
+__version__ = "0.70.0"