{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Downloading and processing of HSC PDR2 wide photo_z\n",
    "\n",
    "We will perform some simple modifications to the tables of HSC PDR2 wide photoz generated from the sql\n",
    "\n",
    "'dmu_products/dmu0/dmu0_HSC/sql/wide_photoz_pdr2.sql'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: AstropyDeprecationWarning: astropy.extern.six will be removed in 4.0, use the six module directly if it is still needed [astropy.extern.six]\n",
      "WARNING:astropy:AstropyDeprecationWarning: astropy.extern.six will be removed in 4.0, use the six module directly if it is still needed\n"
     ]
    }
   ],
   "source": [
    "from astropy.table import Table, Column, vstack\n",
    "from astropy.io.ascii import read\n",
    "\n",
    "from herschelhelp.utils import position_to_field\n",
    "from herschelhelp_internal.utils import coords_to_hpidx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "filenames = [\n",
    "    'HSC-PDR2_wide_W01_WIDE01H',\n",
    "    'HSC-PDR2_wide_W02_XMM',\n",
    "    'HSC-PDR2_wide_W03_GAMA09H',\n",
    "    'HSC-PDR2_wide_W04_WIDE12HpGAMA15H',\n",
    "    'HSC-PDR2_wide_W05_VVDS',\n",
    "    'HSC-PDR2_wide_W06_HECTOMAP',\n",
    "    'HSC-PDR2_wide_W07_AEGIS'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "pdr2 = Table()\n",
    "for f in filenames:\n",
    "    pdr2 = vstack([\n",
    "        pdr2,\n",
    "        read(\"./data/{}.csv\".format(f), header_start=40)\n",
    "    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "pdr2.add_column(Column(\n",
    "    data=coords_to_hpidx(pdr2['ra'], pdr2['dec'], order=13),\n",
    "    name=\"hp_idx\"\n",
    "))\n",
    "\n",
    "pdr2.add_column(Column(\n",
    "    data=position_to_field(pdr2['ra'], pdr2['dec'], dmu_loc='../../'),\n",
    "    name=\"field\"\n",
    "))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<i>Table masked=True length=402759</i>\n",
       "<table id=\"table5881222592\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>object_id</th><th>ra</th><th>dec</th><th>g_cmodel_mag</th><th>g_cmodel_magsigma</th><th>r_cmodel_mag</th><th>r_cmodel_magsigma</th><th>i_cmodel_mag</th><th>i_cmodel_magsigma</th><th>z_cmodel_mag</th><th>z_cmodel_magsigma</th><th>y_cmodel_mag</th><th>y_cmodel_magsigma</th><th>photoz_best</th><th>photoz_std_best</th><th>stellar_mass</th><th>stellar_mass_err68_min</th><th>stellar_mass_err68_max</th><th>sfr</th><th>sfr_err68_min</th><th>sfr_err68_max</th><th>hp_idx</th><th>field</th></tr></thead>\n",
       "<thead><tr><th>int64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>int64</th><th>str18</th></tr></thead>\n",
       "<tr><td>73979566133084180</td><td>213.88265430072687</td><td>51.740363751700805</td><td>23.0946999</td><td>0.020895293</td><td>23.215723</td><td>0.0302032698</td><td>23.016222</td><td>0.0246789604</td><td>23.1860771</td><td>0.0861174613</td><td>23.0555267</td><td>0.1764981</td><td>2.06999993</td><td>0.242393076</td><td>21927999500.0</td><td>11249000400.0</td><td>45937000400.0</td><td>25.5830002</td><td>18.5949993</td><td>34.8199997</td><td>175994725</td><td>EGS</td></tr>\n",
       "<tr><td>73979566133084188</td><td>213.88993383286126</td><td>51.74062309277152</td><td>24.421875</td><td>0.0809714496</td><td>23.8463326</td><td>0.0678118244</td><td>23.2359772</td><td>0.0407232493</td><td>22.7139378</td><td>0.0874182954</td><td>22.9498634</td><td>0.209047094</td><td>0.75</td><td>0.149059072</td><td>3084000000.0</td><td>1698200060.0</td><td>5714400260.0</td><td>0.999899983</td><td>0.502870023</td><td>1.96159995</td><td>175994714</td><td>EGS</td></tr>\n",
       "<tr><td>73979566133084191</td><td>213.89310387374528</td><td>51.74069318520831</td><td>24.9451504</td><td>0.106136128</td><td>24.6018562</td><td>0.126180887</td><td>23.8398609</td><td>0.0637989864</td><td>23.5722313</td><td>0.137099996</td><td>23.6057777</td><td>0.397458017</td><td>0.75999999</td><td>0.237742841</td><td>1292600060.0</td><td>646280000.0</td><td>2471600130.0</td><td>0.526730001</td><td>0.269199997</td><td>0.933170021</td><td>175994714</td><td>EGS</td></tr>\n",
       "<tr><td>73979566133084201</td><td>213.8390870765786</td><td>51.74153430293123</td><td>25.0267334</td><td>0.16844295</td><td>24.5105553</td><td>0.172758877</td><td>22.9234085</td><td>0.06466759</td><td>22.3371315</td><td>0.1094236</td><td>22.8625374</td><td>0.26070413</td><td>0.699999988</td><td>0.120709196</td><td>9022300160.0</td><td>4908299780.0</td><td>15176999900.0</td><td>0.002905</td><td>9.99999975e-05</td><td>0.234410003</td><td>175994730</td><td>EGS</td></tr>\n",
       "<tr><td>73979566133084202</td><td>213.8355615812424</td><td>51.74155533107493</td><td>nan</td><td>nan</td><td>26.7066956</td><td>1.07895362</td><td>24.7455769</td><td>0.177560493</td><td>24.9092617</td><td>0.672663271</td><td>nan</td><td>nan</td><td>3.82999992</td><td>2.17150688</td><td>1604899970.0</td><td>266770000.0</td><td>16081999900.0</td><td>0.151659995</td><td>0.00010094</td><td>17.2380009</td><td>175994730</td><td>EGS</td></tr>\n",
       "<tr><td>73979566133084203</td><td>213.8675269662762</td><td>51.74135205942915</td><td>25.8426399</td><td>0.258565634</td><td>nan</td><td>nan</td><td>24.8253403</td><td>0.151029304</td><td>24.8391209</td><td>0.52028352</td><td>nan</td><td>nan</td><td>0.949999988</td><td>1.51725125</td><td>1214000000.0</td><td>295609984.0</td><td>7656600060.0</td><td>0.490269989</td><td>0.102590002</td><td>6.19810009</td><td>175994726</td><td>EGS</td></tr>\n",
       "<tr><td>73979566133084209</td><td>213.81206801351166</td><td>51.74241749339296</td><td>23.1212177</td><td>0.0419582464</td><td>23.1707287</td><td>0.0504072383</td><td>22.4334068</td><td>0.032119412</td><td>21.6228867</td><td>0.0551997349</td><td>22.6319923</td><td>0.211143062</td><td>0.879999995</td><td>0.0869011059</td><td>6984099840.0</td><td>4160800000.0</td><td>11619000300.0</td><td>3.89310002</td><td>2.77150011</td><td>6.06150007</td><td>175994774</td><td>EGS</td></tr>\n",
       "<tr><td>73979566133084212</td><td>213.8238972739577</td><td>51.74222823684063</td><td>27.7403069</td><td>1.28472364</td><td>26.1703224</td><td>0.425316006</td><td>24.9703751</td><td>0.142886072</td><td>24.668644</td><td>0.356534123</td><td>23.9730816</td><td>0.373745531</td><td>0.910000026</td><td>0.3092857</td><td>1752999940.0</td><td>432940000.0</td><td>7976999940.0</td><td>0.00728339981</td><td>9.99999975e-05</td><td>0.209680006</td><td>175994773</td><td>EGS</td></tr>\n",
       "<tr><td>73979566133084217</td><td>213.80616462201965</td><td>51.742732922741915</td><td>23.0988312</td><td>0.0230495259</td><td>23.0564003</td><td>0.0333248414</td><td>22.8698025</td><td>0.030782124</td><td>22.8383331</td><td>0.0869254246</td><td>23.1027393</td><td>0.219190523</td><td>2.16000009</td><td>0.661843896</td><td>37808001000.0</td><td>14757000200.0</td><td>83242000400.0</td><td>46.2709999</td><td>25.2980003</td><td>66.2060013</td><td>175994780</td><td>EGS</td></tr>\n",
       "<tr><td>73979566133084221</td><td>213.7961624052406</td><td>51.74282404718604</td><td>26.245121</td><td>0.426351219</td><td>25.7339592</td><td>0.401732951</td><td>24.7069817</td><td>0.13982062</td><td>24.3712139</td><td>0.3149589</td><td>26.8592224</td><td>6.34023428</td><td>0.839999974</td><td>1.80468822</td><td>1455399940.0</td><td>324480000.0</td><td>14821000200.0</td><td>0.370599985</td><td>0.0287469998</td><td>17.2299995</td><td>175994777</td><td>EGS</td></tr>\n",
       "<tr><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td></tr>\n",
       "<tr><td>74649168714432205</td><td>213.90152624695293</td><td>53.236817098641545</td><td>nan</td><td>nan</td><td>26.0554924</td><td>1.30071688</td><td>24.3754101</td><td>0.30441612</td><td>23.1159992</td><td>0.236875787</td><td>21.4513359</td><td>0.130216464</td><td>1.13999999</td><td>0.639427364</td><td>31277000700.0</td><td>8135900160.0</td><td>82518999000.0</td><td>0.000100990001</td><td>9.99999975e-05</td><td>0.0499729998</td><td>187222805</td><td>not_help</td></tr>\n",
       "<tr><td>74649168714432208</td><td>213.94103572048908</td><td>53.23756404238926</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>23.6729488</td><td>0.237946481</td><td>22.6605034</td><td>0.227754727</td><td>20.6773949</td><td>0.0949811488</td><td>1.15999997</td><td>0.204146907</td><td>73050996700.0</td><td>28077000700.0</td><td>162479997000.0</td><td>0.000100769998</td><td>9.99999975e-05</td><td>0.0182050001</td><td>187222502</td><td>not_help</td></tr>\n",
       "<tr><td>74649168714432209</td><td>213.94057254362943</td><td>53.237462515289195</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>23.862196</td><td>0.239833906</td><td>23.4570198</td><td>0.403398007</td><td>20.9901428</td><td>0.107366525</td><td>1.22000003</td><td>0.646059334</td><td>45245001700.0</td><td>10387000300.0</td><td>131899998000.0</td><td>0.000100860001</td><td>9.99999975e-05</td><td>0.027152</td><td>187222502</td><td>not_help</td></tr>\n",
       "<tr><td>74649168714432215</td><td>213.88163570000916</td><td>53.23754953850307</td><td>25.9300861</td><td>2.09401798</td><td>25.194437</td><td>2.62530327</td><td>21.2164345</td><td>0.0769277066</td><td>20.3028774</td><td>0.0785751939</td><td>19.1046677</td><td>0.0777843595</td><td>0.860000014</td><td>0.09886536</td><td>203970003000.0</td><td>138589995000.0</td><td>313040011000.0</td><td>0.000100700003</td><td>9.99999975e-05</td><td>0.0443679988</td><td>187222806</td><td>not_help</td></tr>\n",
       "<tr><td>74649168714432217</td><td>213.8808336047542</td><td>53.23741900374859</td><td>nan</td><td>nan</td><td>23.5566978</td><td>1.18840623</td><td>21.9196854</td><td>0.0929267928</td><td>20.7375984</td><td>0.0744943917</td><td>18.8998318</td><td>0.0728724003</td><td>0.970000029</td><td>0.0550813861</td><td>232539996000.0</td><td>181350007000.0</td><td>309029994000.0</td><td>0.000100739999</td><td>9.99999975e-05</td><td>0.0483709984</td><td>187222806</td><td>not_help</td></tr>\n",
       "<tr><td>74649168714432220</td><td>213.9410091010875</td><td>53.238680856384</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>24.6577587</td><td>0.346870214</td><td>23.6515694</td><td>0.338777244</td><td>21.4614449</td><td>0.111151174</td><td>1.23000002</td><td>0.782817304</td><td>35582001200.0</td><td>7976500220.0</td><td>97393000400.0</td><td>0.00010089</td><td>9.99999975e-05</td><td>0.0300360005</td><td>187222502</td><td>not_help</td></tr>\n",
       "<tr><td>74649168714432221</td><td>213.94107345674502</td><td>53.23919576096229</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>24.899292</td><td>0.306392878</td><td>24.0103092</td><td>0.335350603</td><td>21.9399414</td><td>0.131637022</td><td>1.25</td><td>1.04977143</td><td>23094999000.0</td><td>3912100100.0</td><td>74461003800.0</td><td>0.000101019999</td><td>9.99999975e-05</td><td>0.0497040004</td><td>187222502</td><td>not_help</td></tr>\n",
       "<tr><td>74649168714432222</td><td>213.94038874078416</td><td>53.238535813959004</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>24.2435169</td><td>0.291008115</td><td>23.0543766</td><td>0.233248711</td><td>21.2105923</td><td>0.106926374</td><td>1.21000004</td><td>0.403665692</td><td>57166000100.0</td><td>19510999000.0</td><td>130579997000.0</td><td>0.00010071</td><td>9.99999975e-05</td><td>0.0112720001</td><td>187222502</td><td>not_help</td></tr>\n",
       "<tr><td>74649168714432300</td><td>213.85980544929234</td><td>53.12456104583777</td><td>24.2378254</td><td>0.0803623721</td><td>24.1490364</td><td>0.165776685</td><td>24.7114639</td><td>0.242692739</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>2.23000002</td><td>0.509269238</td><td>6693000190.0</td><td>2764800000.0</td><td>17171000300.0</td><td>7.71210003</td><td>3.9849999</td><td>15.948</td><td>187222241</td><td>not_help</td></tr>\n",
       "<tr><td>74649168714432301</td><td>213.86022593954567</td><td>53.12411261063162</td><td>23.5425034</td><td>0.0832843408</td><td>23.4333248</td><td>0.15879494</td><td>24.2459564</td><td>0.378139973</td><td>23.8623371</td><td>0.605079114</td><td>25.6364956</td><td>7.45226765</td><td>1.83000004</td><td>0.538736522</td><td>13532000300.0</td><td>5233399810.0</td><td>35404001300.0</td><td>13.4250002</td><td>6.64839983</td><td>29.9769993</td><td>187222241</td><td>not_help</td></tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Table masked=True length=402759>\n",
       "    object_id             ra                dec         ...   hp_idx   field  \n",
       "      int64            float64            float64       ...   int64    str18  \n",
       "----------------- ------------------ ------------------ ... --------- --------\n",
       "73979566133084180 213.88265430072687 51.740363751700805 ... 175994725      EGS\n",
       "73979566133084188 213.88993383286126  51.74062309277152 ... 175994714      EGS\n",
       "73979566133084191 213.89310387374528  51.74069318520831 ... 175994714      EGS\n",
       "73979566133084201  213.8390870765786  51.74153430293123 ... 175994730      EGS\n",
       "73979566133084202  213.8355615812424  51.74155533107493 ... 175994730      EGS\n",
       "73979566133084203  213.8675269662762  51.74135205942915 ... 175994726      EGS\n",
       "73979566133084209 213.81206801351166  51.74241749339296 ... 175994774      EGS\n",
       "73979566133084212  213.8238972739577  51.74222823684063 ... 175994773      EGS\n",
       "73979566133084217 213.80616462201965 51.742732922741915 ... 175994780      EGS\n",
       "73979566133084221  213.7961624052406  51.74282404718604 ... 175994777      EGS\n",
       "              ...                ...                ... ...       ...      ...\n",
       "74649168714432205 213.90152624695293 53.236817098641545 ... 187222805 not_help\n",
       "74649168714432208 213.94103572048908  53.23756404238926 ... 187222502 not_help\n",
       "74649168714432209 213.94057254362943 53.237462515289195 ... 187222502 not_help\n",
       "74649168714432215 213.88163570000916  53.23754953850307 ... 187222806 not_help\n",
       "74649168714432217  213.8808336047542  53.23741900374859 ... 187222806 not_help\n",
       "74649168714432220  213.9410091010875    53.238680856384 ... 187222502 not_help\n",
       "74649168714432221 213.94107345674502  53.23919576096229 ... 187222502 not_help\n",
       "74649168714432222 213.94038874078416 53.238535813959004 ... 187222502 not_help\n",
       "74649168714432300 213.85980544929234  53.12456104583777 ... 187222241 not_help\n",
       "74649168714432301 213.86022593954567  53.12411261063162 ... 187222241 not_help"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pdr2.write('./data/HSC-PDR2_wide_full.fits', overwrite = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python (herschelhelp_internal)",
   "language": "python",
   "name": "helpint"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}