{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# UKIRT Hemisphere Survey (UHS)\n",
    "\n",
    "This notebook takes in the downloaded survey and filters it per field creating new fits files for each field."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import subprocess\n",
    "\n",
    "from astropy.table import Table\n",
    "from astropy import units as u\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from pymoc import MOC\n",
    "\n",
    "from herschelhelp_internal.utils import inMoc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#TODO: Maybe check for edge effects and add add extra leeway\n",
    "fields= [\n",
    "# Field                RAtyp    DECtyp   RAmin   RAmax   DECmin  DECmax\n",
    "['AKARI-NEP',          274.65,  65.796,  263.6,  266.5,  68.48,  69.51 ],\n",
    "['AKARI-SEP',          72.231, -54.380,   66.1,  75.5,  -55.96, -51.61 ],\n",
    "['Bootes',             216.43,  32.401,  215.7,  220.7,  32.12,  36.18 ],\n",
    "['CDFS-SWIRE',         51.022, -29.818,   50.7,  55.5,  -30.50, -25.92 ],\n",
    "['COSMOS',             149.29,  1.0821,  148.6,  151.6,  0.72,   3.71  ],\n",
    "['EGS',                217.27,  53.644,  212.3,  217.6,  51.08,  54.31 ],\n",
    "['ELAIS-N1',           247.09,  55.175,  237.9,  247.9,  52.37,  57.59 ],\n",
    "['ELAIS-N2',           248.42,  39.127,  246.1,  252.3,  39.02,  43.02 ],\n",
    "['ELAIS-S1',           7.1062, -43.863,    6.3,  11.3,  -45.60, -41.53 ],\n",
    "['GAMA-09',            129.07, -2.2317,  127.1,  142.2, -2.53,   3.55  ],\n",
    "['GAMA-12',            172.84,  -0.482,  172.2,  187.4, -3.54,   2.59  ],\n",
    "['GAMA-15',            211.75, -2.2857,  209.9,  225.3, -2.57,   3.5   ],\n",
    "['HDF-N',              190.25,  62.205,  188.0,  190.4,  61.69,  62.78 ],\n",
    "['Herschel-Stripe-82', 353.75, -7.1089,  348.3,  19.1,  -9.5,    9.25  ],\n",
    "['Lockman-SWIRE',      161.94,  59.056,  154.7,  167.8,  54.95,  60.89 ],\n",
    "['NGP',                192.89,  22.099,  189.8,  209.3,  21.62,  36.16 ],\n",
    "['SA13',               197.89,  42.440,  197.5,  198.6,  42.34,  43.09 ],\n",
    "['SGP',                334.29, -34.503,  333.5,  28.2,  -37.1,  -26.0  ],\n",
    "['SPIRE-NEP',          266.33,  68.790,  264.6,  275.4,  64.41,  68.63 ],\n",
    "['SSDF',               341.57, -59.186,  340.5,  2.8,   -61.2,  -48.6  ],\n",
    "['xFLS',               261.38,  58.018,  255.6,  262.5,  57.85,  60.84 ],\n",
    "['XMM-13hr',           203.31,  37.474,  202.8,  204.4,  37.31,  38.52 ],\n",
    "['XMM-LSS',            32.941, -6.0229,   32.1,  38.2,  -7.53,  -1.52  ]]\n",
    "   \n",
    "    \n",
    "#Has a field been removed?  ( RA BETWEEN 13.4 AND 36.3 AND DEC BETWEEN -2.32 AND 2.49 )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: UnitsWarning: 'RADIANS' did not parse as fits unit: At col 0, Unit 'RADIANS' not supported by the FITS standard.  [astropy.units.core]\n"
     ]
    }
   ],
   "source": [
    "uhs = Table.read(\"data/UHS-DR1_HELP.fits\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "uhs[\"RA\"].unit = u.rad\n",
    "uhs[\"DEC\"].unit = u.rad\n",
    "uhs[\"RA\"].convert_unit_to(u.deg)\n",
    "uhs[\"DEC\"].convert_unit_to(u.deg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<Table masked=True length=10>\n",
       "<table id=\"table140215789830216-388698\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>idx</th><th>SOURCEID</th><th>CUEVENTID</th><th>FRAMESETID</th><th>RA</th><th>DEC</th><th>CX</th><th>CY</th><th>CZ</th><th>HTMID</th><th>L</th><th>B</th><th>LAMBDA</th><th>ETA</th><th>PRIORSEC</th><th>MERGEDCLASSSTAT</th><th>MERGEDCLASS</th><th>PSTAR</th><th>PGALAXY</th><th>PNOISE</th><th>PSATURATED</th><th>EBV</th><th>AJ</th><th>JHALLMAG</th><th>JHALLMAGERR</th><th>JPETROMAG</th><th>JPETROMAGERR</th><th>JPSFMAG</th><th>JPSFMAGERR</th><th>JSERMAG2D</th><th>JSERMAG2DERR</th><th>JAPERMAG3</th><th>JAPERMAG3ERR</th><th>JAPERMAG4</th><th>JAPERMAG4ERR</th><th>JAPERMAG6</th><th>JAPERMAG6ERR</th><th>JGAUSIG</th><th>JELL</th><th>JPA</th><th>JERRBITS</th><th>JDEBLEND</th><th>JCLASS</th><th>JCLASSSTAT</th><th>JPPERRBITS</th><th>JSEQNUM</th><th>JXI</th><th>JETA</th></tr></thead>\n",
       "<thead><tr><th></th><th></th><th></th><th></th><th>deg</th><th>deg</th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th></tr></thead>\n",
       "<tr><td>0</td><td>459598275336</td><td>46557</td><td>459561520602</td><td>249.091395712</td><td>41.8277092465</td><td>-0.265929128626</td><td>-0.696085867558</td><td>0.666892917592</td><td>15320905220532</td><td>66.0875770096</td><td>42.2764334789</td><td>42.0871194535</td><td>31.4778183978</td><td>459561576148</td><td>-2.10839</td><td>-1</td><td>0.993865</td><td>0.00306749</td><td>0.00306749</td><td>0.0</td><td>0.00753558</td><td>0.00669913</td><td>17.8318</td><td>0.046405</td><td>17.7934</td><td>0.0700357</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>17.7371</td><td>0.0377347</td><td>17.7535</td><td>0.0474782</td><td>17.8746</td><td>0.0965107</td><td>1.22327</td><td>0.185815</td><td>105.997</td><td>0</td><td>--</td><td>-1</td><td>-1.49085</td><td>4194304</td><td>1</td><td>0.0</td><td>0.0</td></tr>\n",
       "<tr><td>1</td><td>459598275337</td><td>46557</td><td>459561520602</td><td>249.090484529</td><td>41.7588548997</td><td>-0.266226030817</td><td>-0.696829786242</td><td>0.665996959093</td><td>15320721515998</td><td>65.994515431</td><td>42.2761564233</td><td>42.142361292</td><td>31.4224044753</td><td>459561576148</td><td>-11.0131</td><td>0</td><td>0.00306749</td><td>0.00306749</td><td>0.993865</td><td>0.0</td><td>0.00664529</td><td>0.00590767</td><td>17.602</td><td>0.029693</td><td>17.5066</td><td>0.0626612</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>17.4364</td><td>0.0293875</td><td>17.4445</td><td>0.0363109</td><td>17.4465</td><td>0.065462</td><td>1.30137</td><td>0.169551</td><td>62.1415</td><td>0</td><td>--</td><td>0</td><td>-7.78744</td><td>4194304</td><td>2</td><td>0.0</td><td>0.0</td></tr>\n",
       "<tr><td>2</td><td>459598275338</td><td>46557</td><td>459561520602</td><td>249.089374961</td><td>41.6795933398</td><td>-0.266568101206</td><td>-0.697684607174</td><td>0.664964387265</td><td>15320734749549</td><td>65.8873879373</td><td>42.2757914461</td><td>42.2058919606</td><td>31.3584436703</td><td>459561576148</td><td>3.93553</td><td>1</td><td>0.00306749</td><td>0.993865</td><td>0.00306749</td><td>0.0</td><td>0.00586879</td><td>0.00521736</td><td>18.6699</td><td>0.132642</td><td>18.8291</td><td>0.241219</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>18.7908</td><td>0.0960274</td><td>18.7726</td><td>0.12005</td><td>18.8296</td><td>0.235783</td><td>1.06449</td><td>0.089888</td><td>59.3155</td><td>0</td><td>--</td><td>1</td><td>2.78284</td><td>4194304</td><td>3</td><td>0.0</td><td>0.0</td></tr>\n",
       "<tr><td>3</td><td>459598275339</td><td>46557</td><td>459561520602</td><td>249.089224553</td><td>41.6707234215</td><td>-0.266606671154</td><td>-0.697780061623</td><td>0.664848756108</td><td>15320734616225</td><td>65.8753992885</td><td>42.2757641249</td><td>42.2129876315</td><td>31.3512565069</td><td>459561576148</td><td>0.675435</td><td>-1</td><td>0.993865</td><td>0.00306749</td><td>0.00306749</td><td>0.0</td><td>0.00583052</td><td>0.00518334</td><td>15.8568</td><td>0.0102708</td><td>15.82</td><td>0.014632</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>15.8035</td><td>0.00881926</td><td>15.7949</td><td>0.00981466</td><td>15.7996</td><td>0.0156522</td><td>1.59317</td><td>0.0841488</td><td>170.574</td><td>0</td><td>--</td><td>-1</td><td>0.477605</td><td>4194304</td><td>4</td><td>0.0</td><td>0.0</td></tr>\n",
       "<tr><td>4</td><td>459598275340</td><td>46557</td><td>459561520602</td><td>249.089307827</td><td>41.7307409658</td><td>-0.26635694576</td><td>-0.697129503289</td><td>0.665630853468</td><td>15320732877909</td><td>65.9565056197</td><td>42.2766225202</td><td>42.1645551696</td><td>31.3990957442</td><td>459561576148</td><td>-1.94055</td><td>-1</td><td>0.993865</td><td>0.00306749</td><td>0.00306749</td><td>0.0</td><td>0.00642404</td><td>0.00571097</td><td>14.2094</td><td>0.00364238</td><td>14.1972</td><td>0.0039325</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>14.1687</td><td>0.00331449</td><td>14.1747</td><td>0.00337262</td><td>14.1766</td><td>0.00426959</td><td>1.70204</td><td>0.0480872</td><td>135.19</td><td>0</td><td>--</td><td>-1</td><td>-1.37217</td><td>4194304</td><td>5</td><td>0.0</td><td>0.0</td></tr>\n",
       "<tr><td>5</td><td>459598275341</td><td>46557</td><td>459561520602</td><td>249.088783225</td><td>41.8386617913</td><td>-0.265915361617</td><td>-0.695954643429</td><td>0.667035347448</td><td>15320905232307</td><td>66.1023436874</td><td>42.2785249258</td><td>42.0771202884</td><td>31.4843866085</td><td>459561576148</td><td>2.12365</td><td>1</td><td>0.00306749</td><td>0.993865</td><td>0.00306749</td><td>0.0</td><td>0.00766715</td><td>0.0068161</td><td>19.4787</td><td>0.331165</td><td>19.4787</td><td>0.331165</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>19.5737</td><td>0.190161</td><td>19.4502</td><td>0.217731</td><td>19.399</td><td>0.389935</td><td>0.814482</td><td>0.0608817</td><td>46.1616</td><td>0</td><td>--</td><td>1</td><td>1.50165</td><td>4194304</td><td>6</td><td>0.0</td><td>0.0</td></tr>\n",
       "<tr><td>6</td><td>459598275342</td><td>46557</td><td>459561520602</td><td>249.088964626</td><td>41.8271266834</td><td>-0.265961083771</td><td>-0.696080917527</td><td>0.6668853411</td><td>15320904958065</td><td>66.0867570644</td><td>42.2782370872</td><td>42.0865237228</td><td>31.4753831739</td><td>459561576148</td><td>-5.07346</td><td>-1</td><td>0.993865</td><td>0.00306749</td><td>0.00306749</td><td>0.0</td><td>0.00752875</td><td>0.00669306</td><td>16.779</td><td>0.020396</td><td>16.7723</td><td>0.0292268</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>16.6917</td><td>0.0163059</td><td>16.711</td><td>0.0196059</td><td>16.7732</td><td>0.035773</td><td>1.43856</td><td>0.0924337</td><td>106.448</td><td>0</td><td>--</td><td>-1</td><td>-3.58748</td><td>4194304</td><td>7</td><td>0.0</td><td>0.0</td></tr>\n",
       "<tr><td>7</td><td>459598275343</td><td>46557</td><td>459561520602</td><td>249.088172488</td><td>41.6673778105</td><td>-0.266633339987</td><td>-0.697811429524</td><td>0.66480513749</td><td>15320734629478</td><td>65.8708613999</td><td>42.2764969406</td><td>42.2152252045</td><td>31.347734619</td><td>459561576148</td><td>-0.125998</td><td>-1</td><td>0.993865</td><td>0.00306749</td><td>0.00306749</td><td>0.0</td><td>0.0058114</td><td>0.00516633</td><td>18.0646</td><td>0.0711287</td><td>18.0819</td><td>0.0991237</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>17.9571</td><td>0.0465055</td><td>17.9765</td><td>0.0591082</td><td>18.1006</td><td>0.121431</td><td>1.35322</td><td>0.0117524</td><td>168.476</td><td>0</td><td>--</td><td>-1</td><td>-0.0890941</td><td>4194304</td><td>8</td><td>0.0</td><td>0.0</td></tr>\n",
       "<tr><td>8</td><td>459598275344</td><td>46557</td><td>459561520602</td><td>249.087726989</td><td>41.8419330134</td><td>-0.265914597489</td><td>-0.695914165265</td><td>0.667077882579</td><td>15320905273802</td><td>66.1067506583</td><td>42.2793546549</td><td>42.0740127768</td><td>31.4861242174</td><td>459561651413</td><td>-1.45288</td><td>-1</td><td>0.993865</td><td>0.00306749</td><td>0.00306749</td><td>0.0</td><td>0.00770535</td><td>0.00685005</td><td>19.8331</td><td>0.39839</td><td>19.7844</td><td>0.365981</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>19.5949</td><td>0.193994</td><td>19.6373</td><td>0.258409</td><td>20.0903</td><td>0.73655</td><td>0.987171</td><td>0.0379558</td><td>97.3524</td><td>0</td><td>--</td><td>-1</td><td>-1.02734</td><td>32768</td><td>9</td><td>0.0</td><td>0.0</td></tr>\n",
       "<tr><td>9</td><td>459598275345</td><td>46557</td><td>459561520602</td><td>249.087532698</td><td>41.7413018145</td><td>-0.266334746293</td><td>-0.69700663074</td><td>0.665768397884</td><td>15320721493672</td><td>65.9707510277</td><td>42.2781029821</td><td>42.1552435962</td><td>31.4060508675</td><td>459561576148</td><td>0.174297</td><td>-1</td><td>0.993865</td><td>0.00306749</td><td>0.00306749</td><td>0.0</td><td>0.00651853</td><td>0.00579497</td><td>16.9461</td><td>0.0254199</td><td>16.9034</td><td>0.0353545</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>-9.99999e+08</td><td>16.8973</td><td>0.0191435</td><td>16.8925</td><td>0.0228294</td><td>16.8646</td><td>0.0389321</td><td>1.50494</td><td>0.0550683</td><td>62.2292</td><td>0</td><td>--</td><td>-1</td><td>0.123247</td><td>32768</td><td>10</td><td>0.0</td><td>0.0</td></tr>\n",
       "</table><style>table.dataTable {clear: both; width: auto !important; margin: 0 !important;}\n",
       ".dataTables_info, .dataTables_length, .dataTables_filter, .dataTables_paginate{\n",
       "display: inline-block; margin-right: 1em; }\n",
       ".paginate_button { margin-right: 5px; }\n",
       "</style>\n",
       "<script>\n",
       "require.config({paths: {\n",
       "    datatables: 'https://cdn.datatables.net/1.10.12/js/jquery.dataTables.min'\n",
       "}});\n",
       "require([\"datatables\"], function(){\n",
       "    console.log(\"$('#table140215789830216-388698').dataTable()\");\n",
       "    $('#table140215789830216-388698').dataTable({\n",
       "        \"order\": [],\n",
       "        \"iDisplayLength\": 50,\n",
       "        \"aLengthMenu\": [[10, 25, 50, 100, 500, 1000, -1], [10, 25, 50, 100, 500, 1000, 'All']],\n",
       "        \"pagingType\": \"full_numbers\"\n",
       "    });\n",
       "});\n",
       "</script>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uhs[:10].show_in_notebook()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "log_file = open('log.txt', 'a')\n",
    "for field in fields:\n",
    "    field_moc = MOC(filename=\"../../dmu2/dmu2_field_coverages/{}_MOC.fits\".format(field[0]))\n",
    "    field_mask= inMoc(uhs[\"RA\"],uhs[\"DEC\"],field_moc)\n",
    "    if not np.all( np.logical_not(field_mask)):\n",
    "        uhs[field_mask].write(\"data/UHS-DR1_{}.fits\".format(field[0]), overwrite=True)\n",
    "    \n",
    "        subprocess.call([\"stilts\", \n",
    "                 \"pixfoot\", \n",
    "                 \"in=./data/UHS-DR1_{}.fits\".format(field[0]), \n",
    "                 \"out=./data/UHS-DR1_{}_MOC.fits\".format(field[0]),\n",
    "                 \"ra=RA\",\n",
    "                 \"dec=DEC\",\n",
    "                 \"order=10\"], \n",
    "                    stdout=log_file)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:herschelhelp_internal]",
   "language": "python",
   "name": "conda-env-herschelhelp_internal-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}