QA-Check numbers.ipynb

{
 "cells": [
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "import sys\n",
    "sys.path.insert(0, \"/home/yymao/Dropbox/Academia/Collaborations/SAGA-local/saga\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import SAGA\n",
    "from easyquery import Query, QueryMaker\n",
    "from SAGA.database import FitsTable\n",
    "from SAGA import ObjectCuts as C\n",
    "from SAGA import HostCuts as H\n",
    "\n",
    "from itertools import combinations\n",
    "from collections import Counter, defaultdict\n",
    "from astropy.table import Table, join, vstack\n",
    "from astropy.coordinates import SkyCoord, Distance\n",
    "from SAGA.utils import add_skycoord\n",
    "from easyquery import Query, QueryMaker\n",
    "from astropy.io import ascii\n",
    "\n",
    "import SAGA\n",
    "from SAGA.database import FitsTable, GoogleSheets\n",
    "from SAGA import ObjectCuts as C\n",
    "from SAGA.utils import add_skycoord, fill_values_by_query\n",
    "from SAGA.utils.distance import z2v, d2m, m2d\n",
    "from SAGA.utils.display import show_images\n",
    "from SAGA.objects.object_catalog import calc_fiducial_p_sat, calc_fiducial_p_sat_corrected\n",
    "from scipy.stats import spearmanr\n",
    "print(SAGA.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "saga = SAGA.QuickStart()\n",
    "saga.set_default_base_version(\"paper2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "masterlist = saga.host_catalog.load_master_list()\n",
    "hostlist = saga.host_catalog.load(include_stats=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def annotate_base(base):\n",
    "    base[\"Mr_\"] = base[\"r_mag\"] - d2m(base[\"HOST_DIST\"])\n",
    "    base[\"Mr\"] = np.where(np.isfinite(base[\"Mr\"]) & C.has_spec.mask(base), base[\"Mr\"], base[\"Mr_\"])\n",
    "    base[\"log_sm\"] = np.where(np.isfinite(base[\"log_sm\"]), base[\"log_sm\"], 1.254 + 1.0976 * base[\"gr\"] - 0.4 * base[\"Mr\"])\n",
    "    \n",
    "    base[\"human_selected\"] = 0\n",
    "    hs = saga.database[\"human_selected\"].read()\n",
    "    for i in range(1, 4):\n",
    "        base[\"human_selected\"] += np.in1d(base[\"OBJID\"], Query(f\"score == {i}\").filter(hs, \"OBJID\")).astype(np.int) * i\n",
    "\n",
    "    return base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "base = SAGA.database.FitsTable(\"/home/yymao/Documents/Research/SAGA/PaperII/data-archive/saga_base_all.fits\").read()\n",
    "base = saga.host_catalog.construct_host_query(\"paper2_complete\").filter(base)\n",
    "base = annotate_base(base)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(C.is_sat.count(base), Query(C.is_sat, C.r_abs_limit).count(base))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\n",
    "    Query(C.is_sat, ~C.paper2_targeting_cut).count(base), \n",
    "    Query(\"RHOST_KPC > 10\", C.has_spec, \"SPEC_Z > 0.002\", \"SPEC_Z < 0.015\", ~C.paper2_targeting_cut).count(base),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(C.is_sat.count(base), len(set(Query(C.is_sat).filter(base, \"OBJID\"))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " The inner distance and velocity limits exclude 80 hosts in fields"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nearbr_host_criteria = [\n",
    "    \"abs(GLAT) >= 25\",\n",
    "    \"BRIGHTEST_STAR_R1 >= 5\",\n",
    "    \"M_HALO < 13\",\n",
    "    QueryMaker.equals(\"REMOVED_BY_HAND\", \"False\"),\n",
    "    \"BRIGHTEST_K_R1 >= K_TC + 1.6\",\n",
    "    \"K_ABS >= -24.6\",\n",
    "    \"K_ABS <= -23.0\",\n",
    "]\n",
    "\n",
    "has_ls = Query(\"COVERAGE_DECALS_DR6 >= 0.99\") | \"COVERAGE_DECALS_DR7 >= 0.99\"\n",
    "has_des = Query(\"COVERAGE_DES_DR1 >= 0.99\")\n",
    "has_sdss = Query(\"COVERAGE_SDSS >= 0.99\")\n",
    "has_any = (has_ls | has_des | has_sdss)\n",
    "\n",
    "nearbr_hosts = Query(Query(\"DIST < 25\") | \"V_HELIO < 1400\", has_any, *nearbr_host_criteria).filter(masterlist)\n",
    "print(len(nearbr_hosts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lv_hosts = [\"NGC1023\", \"NGC1156\", \"NGC2903\", \"NGC4258\", \"NGC4565\", \"NGC4631\", \"NGC5023\", \"NGC5194\", \"NGC4826\", \"NGC4594\", \"NGC4736\"]\n",
    "lv_hosts1 = QueryMaker.in1d(\"COMMON_NAME\", lv_hosts).filter(masterlist)\n",
    "assert len(lv_hosts1) == len(lv_hosts)\n",
    "\n",
    "for q in nearbr_host_criteria:\n",
    "    lv_hosts1[str(q)] = Query(q).mask(lv_hosts1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Query(\"COVERAGE_SDSS < 0.9\", H.paper2_complete).filter(hostlist)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These nearby ($<25$\\,Mpc) hosts that also satisfy our stellar mass and environments cuts include the well-studied Milky Way analogs M94 \\citep{Smercina2018}, as well as 3 of the 10 hosts (NGC\\,1023, NGC\\,2903, NGC\\,4258) presented in \\citet{Carlsten2019}. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Query(nearbr_host_criteria[2], nearbr_host_criteria[4], *nearbr_host_criteria[-2:]).filter(lv_hosts1, \"COMMON_NAME\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The definition of our primary targeting region effectively removes galaxy targets beyond $z \\sim 0.3$, while including all galaxies within $z < 0.015$ (and hence all SAGA satellites).\n",
    "In the slightly higher redshift range of $0.015 < z < 0.03$, 83\\% of galaxies are still in the primary targeting region."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Query(C.basic_cut2, \"SPEC_Z < 0.015\", \"SPEC_Z > 0.002\", C.has_spec, ~C.paper2_targeting_cut, \"SATS != 3\").count(base)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Query(C.basic_cut2, \"SPEC_Z >= 0.015\", \"SPEC_Z < 0.03\", C.has_spec, C.paper2_targeting_cut).count(base) / Query(C.basic_cut2, \"SPEC_Z >= 0.015\", \"SPEC_Z < 0.03\", C.has_spec).count(base)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Among \\nhoststotal{} systems, \\nhosts{} have spectroscopic coverage higher than 80\\% (and 22 have coverage higher than 90\\%). We have obtained some SAGA redshifts for an additional 19 systems"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Query(\"paper2_need_spec / paper2_total < 0.1\", H.good).count(hostlist), Query(\"paper2_need_spec / paper2_total < 0.2\", H.good).count(hostlist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Query(\"paper2_need_spec / paper2_total > 0.2\", \"specs_ours_rvir > 50\", H.good).count(hostlist)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "in the primary targeting region, where we find \\nsats{} satellites among 5,419 redshifts. Outside of the primary targeting region, we find no satellites among 21,310 redshifts."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Query(C.basic_cut2, C.has_spec, C.paper2_targeting_cut).count(base)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Query(C.basic_cut2, C.has_spec, ~C.paper2_targeting_cut).count(base)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "we visually inspected targets in SAGA field to identify all dLSBGs and found \\checknumber{70} of them that we have not obtained redshifts (down to $M_{r,o} < -12.3$), and \\checknumber{19}\\% of them are slightly outside the primary targeting region."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Query(C.basic_cut2, ~C.has_spec, \"Mr_ < -12.295\", \"human_selected > 0\").count(base), Query(C.basic_cut2, ~C.has_spec, \"Mr_ < -12.295\", \"human_selected > 0\", ~C.paper2_targeting_cut).count(base)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Query(C.is_sat, C.r_abs_limit, \"EW_Halpha < 2\").count(base)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Specifically, the relative fraction of SAGA to literature redshifts is about 46\\% for bright galaxies ($r_o < 19$) and 82\\% for faint galaxies ($r_o > 19$). \n",
    "\n",
    "For galaxies dimmer than $r_o=17.5$ (19), 77\\% (82\\%) of those redshifts are taken by SAGA."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Query(C.has_spec, C.basic_cut2, C.has_our_specs_only, \"r_mag > 19\").count(base) / Query(C.has_spec, C.basic_cut2, \"r_mag > 19\").count(base)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Query(C.has_spec, C.basic_cut2, C.has_our_specs_only, \"r_mag < 19\").count(base) / Query(C.has_spec, C.basic_cut2, \"r_mag < 19\").count(base)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Query(C.has_spec, C.basic_cut2, C.has_our_specs_only, \"r_mag > 17.5\").count(base) / Query(C.has_spec, C.basic_cut2, \"r_mag > 17.5\").count(base)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "within the primary targeting region and between 10--300 kpc to each host in projection is 160 per host on average ($\\sim$\\,200 targets per square degree)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Query(C.basic_cut2, \"RHOST_KPC > 10\", C.paper2_targeting_cut).count(base) / 36"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Query(C.basic_cut2, \"RHOST_KPC > 10\", C.paper2_targeting_cut).count(base) / 29.1787365"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Among the fainter half of the satellites (54 satellites with $-15.7 < M_{r,o} < -12.3$), $53/62$ (87\\%) have the first redshift measurement from SAGA. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Query(C.is_sat, C.r_abs_limit,  C.has_our_specs_only).count(base) / Query(C.is_sat, C.r_abs_limit).count(base)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.median(Query(C.is_sat, C.r_abs_limit).filter(base, \"Mr\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Query(C.is_sat, C.r_abs_limit, \"Mr >= -15.77\", C.has_our_specs_only).count(base), Query(C.is_sat, C.r_abs_limit, \"Mr >= -15.77\").count(base)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Query(C.basic_cut2, C.has_our_specs_only, ~saga.host_catalog.construct_host_query(\"paper1_complete\")).count(base) / 28"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LMC-like sats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum((k >= 2 for k in Counter(Query(C.is_sat, \"Mr < -17\").filter(base,\"HOSTID\")).values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum((k >= 2 for k in Counter(Query(C.is_sat, \"log_sm > 8.5\").filter(base,\"HOSTID\")).values())) / 36"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum((k >= 1 for k in Counter(Query(C.is_sat, \"log_sm > 9\").filter(base,\"HOSTID\")).values())) / 36"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum((k >= 1 for k in Counter(Query(C.is_sat, \"Mr < -18.3\").filter(base,\"HOSTID\")).values())) / 36"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "spearmanr(*np.array([(t[\"Mr\"].min(), C.r_abs_limit.count(t)) for t in C.is_sat.filter(base, [\"Mr\", \"HOSTID\"]).group_by([\"HOSTID\"]).groups]).T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "spearmanr(*np.array([(t[\"Mr\"].min(), t[\"HOST_MK\"][0]) for t in C.is_sat.filter(base, [\"Mr\", \"HOSTID\", \"HOST_MK\"]).group_by([\"HOSTID\"]).groups]).T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "chosts = H.paper2_complete.filter(hostlist)\n",
    "print(len(chosts))\n",
    "spearmanr(chosts[\"K_ABS\"], chosts[\"sats_Mr_limit\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.median(Query(C.is_sat, C.has_our_specs_only).filter(base, \"Mr\")), np.median(Query(C.is_sat, ~C.has_our_specs_only).filter(base, \"Mr\")), "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum((k >= 1 for k in Counter(Query(C.is_sat, C.r_abs_limit, \"RHOST_KPC < 150\").filter(base,\"HOSTID\")).values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}