{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ELAIS-N1 master catalogue: Flags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from astropy.table import Table\n",
    "\n",
    "import itertools\n",
    "\n",
    "from herschelhelp_internal.flagging import flag_outliers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "SUFFIX = \"20171016\"\n",
    "FIELD = \"ELAIS-N1\"\n",
    "catname = \"../../dmu1/dmu1_ml_ELAIS-N1/data/master_catalogue_elais-n1_{}.fits\".format(SUFFIX)\n",
    "master_catalogue = Table.read(catname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "u_bands = [\"WFC u\", \"Megacam u\"]\n",
    "g_bands = [\"WFC g\", \"Megacam g\", \"SUPRIME g\", \"GPC1 g\"]\n",
    "r_bands = [\"WFC r\", \"Megacam r\", \"SUPRIME r\", \"GPC1 r\"]\n",
    "i_bands = [\"WFC i\",              \"SUPRIME i\", \"GPC1 i\"]\n",
    "z_bands = [\"WFC z\", \"Megacam z\", \"SUPRIME z\", \"GPC1 z\"]\n",
    "y_bands = [                      \"SUPRIME y\", \"GPC1 y\"]\n",
    "\n",
    "all_bands = [u_bands, g_bands, r_bands, i_bands, z_bands, y_bands]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Magnitudes and magnitude erros"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def flag_mag(flagcol, mask):\n",
    "    \n",
    "    # Add flag columns if does not exist\n",
    "    if flagcol not in master_catalogue.colnames:\n",
    "        master_catalogue[flagcol] = np.zeros(len(master_catalogue), dtype=bool)\n",
    "    \n",
    "    # Flagged\n",
    "    master_catalogue[flagcol][mask] = np.ones(len(mask), dtype=bool)\n",
    "    print('    Number of flagged objects:', len(master_catalogue[flagcol][mask]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.a Pan-STARRS Aperture and Total magnitude errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPC1 g\n",
      "  Aperture magnitude\n",
      "    Number of flagged objects: 107\n",
      "  Total magnitude\n",
      "    Number of flagged objects: 0\n",
      "GPC1 r\n",
      "  Aperture magnitude\n",
      "    Number of flagged objects: 174\n",
      "  Total magnitude\n",
      "    Number of flagged objects: 4\n",
      "GPC1 i\n",
      "  Aperture magnitude\n",
      "    Number of flagged objects: 150\n",
      "  Total magnitude\n",
      "    Number of flagged objects: 0\n",
      "GPC1 z\n",
      "  Aperture magnitude\n",
      "    Number of flagged objects: 70\n",
      "  Total magnitude\n",
      "    Number of flagged objects: 1\n",
      "GPC1 y\n",
      "  Aperture magnitude\n",
      "    Number of flagged objects: 37\n",
      "  Total magnitude\n",
      "    Number of flagged objects: 1\n"
     ]
    }
   ],
   "source": [
    "## dmu0: Pan-STARRS stack cat \n",
    "gpc1_err = 0.05000000074505806\n",
    "bands = [\"GPC1 g\", \"GPC1 r\", \"GPC1 i\", \"GPC1 z\", \"GPC1 y\"]\n",
    "\n",
    "for i, band in enumerate(bands):\n",
    "    print(band)\n",
    "    basecol = band.replace(\" \", \"_\").lower()\n",
    "    \n",
    "    ecol_ap, ecol_tot = \"merr_ap_{}\".format(basecol), \"merr_{}\".format(basecol)\n",
    "    flagcol_ap, flagcol_tot = \"flag_ap_{}\".format(basecol), \"flag_{}\".format(basecol)\n",
    "    \n",
    "    mask_ap  = np.where(master_catalogue[ecol_ap]  == gpc1_err)[0]\n",
    "    mask_tot = np.where(master_catalogue[ecol_tot] == gpc1_err)[0]\n",
    "    \n",
    "    print('  Aperture magnitude')\n",
    "    flag_mag(flagcol_ap, mask_ap)\n",
    "    print('  Total magnitude')\n",
    "    flag_mag(flagcol_tot, mask_tot)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.c IRAC Aperture magnitudes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IRAC i1\n",
      "  Aperture magnitude\n",
      "    Number of flagged objects: 0\n",
      "IRAC i2\n",
      "  Aperture magnitude\n",
      "    Number of flagged objects: 0\n",
      "IRAC i3\n",
      "  Aperture magnitude\n",
      "    Number of flagged objects: 0\n",
      "IRAC i4\n",
      "  Aperture magnitude\n",
      "    Number of flagged objects: 0\n"
     ]
    }
   ],
   "source": [
    "irac_mag = 3.9000000001085695\n",
    "bands = [\"IRAC i1\", \"IRAC i2\", \"IRAC i3\", \"IRAC i4\"]\n",
    "\n",
    "for i, band in enumerate(bands):\n",
    "    print(band)\n",
    "    basecol = band.replace(\" \", \"_\").lower()\n",
    "    \n",
    "    ecol_ap = \"merr_ap_{}\".format(basecol)\n",
    "    flagcol_ap = \"flag_ap_{}\".format(basecol)\n",
    "    \n",
    "    mask_ap  = np.where(master_catalogue[ecol_ap]  == irac_mag)[0]\n",
    "    \n",
    "    print('  Aperture magnitude')\n",
    "    flag_mag(flagcol_ap, mask_ap)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Outliers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Megacam u (aperture) - WFC u (aperture):\n",
      "  Number of outliers: 0\n",
      "Megacam u (total) - WFC u (total):\n",
      "  Number of outliers: 0\n",
      "Megacam g (aperture) - WFC g (aperture):\n",
      "  Number of outliers: 0\n",
      "Megacam g (total) - WFC g (total):\n",
      "  Number of outliers: 35\n",
      "SUPRIME g (aperture) - WFC g (aperture):\n",
      "  Number of outliers: 0\n",
      "SUPRIME g (total) - WFC g (total):\n",
      "  Number of outliers: 6\n",
      "GPC1 g (aperture) - WFC g (aperture):\n",
      "  Number of outliers: 0\n",
      "GPC1 g (total) - WFC g (total):\n",
      "  Number of outliers: 0\n",
      "SUPRIME g (aperture) - Megacam g (aperture):\n",
      "  Number of outliers: 1\n",
      "SUPRIME g (total) - Megacam g (total):\n",
      "  Number of outliers: 0\n",
      "GPC1 g (aperture) - Megacam g (aperture):\n",
      "  Number of outliers: 0\n",
      "GPC1 g (total) - Megacam g (total):\n",
      "  Number of outliers: 752\n",
      "GPC1 g (aperture) - SUPRIME g (aperture):\n",
      "  Number of outliers: 0\n",
      "GPC1 g (total) - SUPRIME g (total):\n",
      "  Number of outliers: 23\n",
      "Megacam r (aperture) - WFC r (aperture):\n",
      "  Number of outliers: 0\n",
      "Megacam r (total) - WFC r (total):\n",
      "  Number of outliers: 22\n",
      "SUPRIME r (aperture) - WFC r (aperture):\n",
      "  Number of outliers: 0\n",
      "SUPRIME r (total) - WFC r (total):\n",
      "  Number of outliers: 6\n",
      "GPC1 r (aperture) - WFC r (aperture):\n",
      "  Number of outliers: 0\n",
      "GPC1 r (total) - WFC r (total):\n",
      "  Number of outliers: 0\n",
      "SUPRIME r (aperture) - Megacam r (aperture):\n",
      "  Number of outliers: 0\n",
      "SUPRIME r (total) - Megacam r (total):\n",
      "  Number of outliers: 0\n",
      "GPC1 r (aperture) - Megacam r (aperture):\n",
      "  Number of outliers: 0\n",
      "GPC1 r (total) - Megacam r (total):\n",
      "  Number of outliers: 584\n",
      "GPC1 r (aperture) - SUPRIME r (aperture):\n",
      "  Number of outliers: 0\n",
      "GPC1 r (total) - SUPRIME r (total):\n",
      "  Number of outliers: 31\n",
      "SUPRIME i (aperture) - WFC i (aperture):\n",
      "  Number of outliers: 0\n",
      "SUPRIME i (total) - WFC i (total):\n",
      "  Number of outliers: 0\n",
      "GPC1 i (aperture) - WFC i (aperture):\n",
      "  Number of outliers: 0\n",
      "GPC1 i (total) - WFC i (total):\n",
      "  Number of outliers: 0\n",
      "GPC1 i (aperture) - SUPRIME i (aperture):\n",
      "  Number of outliers: 0\n",
      "GPC1 i (total) - SUPRIME i (total):\n",
      "  Number of outliers: 161\n",
      "Megacam z (aperture) - WFC z (aperture):\n",
      "  Number of outliers: 0\n",
      "Megacam z (total) - WFC z (total):\n",
      "  Number of outliers: 0\n",
      "SUPRIME z (aperture) - WFC z (aperture):\n",
      "  Number of outliers: 0\n",
      "SUPRIME z (total) - WFC z (total):\n",
      "  Number of outliers: 0\n",
      "GPC1 z (aperture) - WFC z (aperture):\n",
      "  Number of outliers: 0\n",
      "GPC1 z (total) - WFC z (total):\n",
      "  Number of outliers: 0\n",
      "SUPRIME z (aperture) - Megacam z (aperture):\n",
      "  Number of outliers: 6\n",
      "SUPRIME z (total) - Megacam z (total):\n",
      "  Number of outliers: 0\n",
      "GPC1 z (aperture) - Megacam z (aperture):\n",
      "  Number of outliers: 0\n",
      "GPC1 z (total) - Megacam z (total):\n",
      "  Number of outliers: 1\n",
      "GPC1 z (aperture) - SUPRIME z (aperture):\n",
      "  Number of outliers: 0\n",
      "GPC1 z (total) - SUPRIME z (total):\n",
      "  Number of outliers: 28\n",
      "GPC1 y (aperture) - SUPRIME y (aperture):\n",
      "  Number of outliers: 0\n",
      "GPC1 y (total) - SUPRIME y (total):\n",
      "  Number of outliers: 9\n"
     ]
    }
   ],
   "source": [
    "for band_of_a_kind in all_bands:\n",
    "    for band1, band2 in itertools.combinations(band_of_a_kind, 2):\n",
    "        #print(band1, band2)\n",
    "\n",
    "        basecol1, basecol2 = band1.replace(\" \", \"_\").lower(), band2.replace(\" \", \"_\").lower()\n",
    "        \n",
    "        # Aperture mag\n",
    "        col1, col2 = \"m_ap_{}\".format(basecol1), \"m_ap_{}\".format(basecol2)\n",
    "        ecol1, ecol2 = \"merr_ap_{}\".format(basecol1), \"merr_ap_{}\".format(basecol2)\n",
    "        flagcol1, flagcol2 = \"flag_ap_{}\".format(basecol1), \"flag_ap_{}\".format(basecol2)\n",
    "        \n",
    "        try:\n",
    "            master_catalogue = flag_outliers(master_catalogue, col1, col2,\n",
    "                      ecol1, ecol2,\n",
    "                      flagcol1, flagcol2,\n",
    "                      labels=(\"{} (aperture)\".format(band1), \"{} (aperture)\".format(band2)))\n",
    "        except KeyError:\n",
    "            print(\"One of {} and {} is not present in the catalogue.\".format(col1, col2))\n",
    "                      \n",
    "        \n",
    "        # Tot mag\n",
    "        col1, col2 = \"m_{}\".format(basecol1), \"m_{}\".format(basecol2)              \n",
    "        ecol1, ecol2 = \"merr_{}\".format(basecol1), \"merr_{}\".format(basecol2)              \n",
    "        flagcol1, flagcol2 = \"flag_{}\".format(basecol1), \"flag_{}\".format(basecol2)\n",
    "              \n",
    "        try:\n",
    "            master_catalogue = flag_outliers(master_catalogue, col1, col2, \n",
    "                      ecol1, ecol2,\n",
    "                      flagcol1, flagcol2,\n",
    "                      labels=(\"{} (total)\".format(band1), \"{} (total)\".format(band2)))   \n",
    "        except KeyError:\n",
    "            print(\"One of {} and {} is not present in the catalogue.\".format(col1, col2))             \n",
    "                      "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Save table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "#Merge any aperture flags#Merge  \n",
    "for col in master_catalogue.colnames:\n",
    "    if col.startswith(\"flag_ap_\"):\n",
    "        try:\n",
    "            master_catalogue[col.replace(\"_ap_\", \"_\")] = (master_catalogue[col.replace(\"_ap_\", \"_\")] |\n",
    "                                                          master_catalogue[col])\n",
    "            master_catalogue.remove_column(col)\n",
    "        except KeyError:\n",
    "            print(\"{} only has aperture flags.\".format(col))\n",
    "            master_catalogue.rename_column(col, col.replace(\"_ap_\", \"_\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "flag_cols = [\"help_id\"]\n",
    "for col in master_catalogue.colnames:\n",
    "    if col.startswith(\"flag_\"):\n",
    "        flag_cols += [col]\n",
    "new_catname = \"./data/{}_{}_flags.fits\".format(FIELD.lower(),SUFFIX)\n",
    "master_catalogue[flag_cols].write(new_catname, overwrite = True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (herschelhelp_internal)",
   "language": "python",
   "name": "helpint"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}