{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# EGS master catalogue: Flags" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import numpy as np\n", "from astropy.table import Table\n", "\n", "import itertools\n", "\n", "from herschelhelp_internal.flagging import flag_outliers\n" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": true }, "outputs": [], "source": [ "SUFFIX = \"20180501\"\n", "FIELD = \"EGS\"\n", "catname = \"../../dmu1/dmu1_ml_EGS/data/master_catalogue_egs_{}.fits\".format(SUFFIX)\n", "master_catalogue = Table.read(catname)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": true }, "outputs": [], "source": [ "u_bands = [\"Megacam u\", \"MMT u\"]\n", "g_bands = [\"Megacam g\", \"SUPRIME g\", \"GPC1 g\", \"90Prime g\", \"MMT g\"] \n", "r_bands = [\"Megacam r\", \"SUPRIME r\", \"GPC1 r\", \"90Prime r\"]\n", "i_bands = [\"Megacam i\", \"SUPRIME i\", \"GPC1 i\", \"MMT i\"]\n", "z_bands = [\"Megacam z\", \"SUPRIME z\", \"GPC1 z\", \"Mosaic z\", \"MMT z\"]\n", "y_bands = [ \"SUPRIME y\", \"GPC1 y\"]\n", "\n", "J_bands = [\"WIRCAM J\", \"WIRCS J\", \"UKIDSS J\", \"OMEGA2000 J\"]\n", "H_bands = [\"WIRCAM H\"]\n", "K_bands = [\"WIRCAM Ks\", \"WIRCS K\", \"NEWFIRM K\", \"OMEGA2000 Ks\"]\n", "\n", "all_bands = [u_bands, g_bands, r_bands, i_bands, z_bands, y_bands, J_bands, K_bands]" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# No aperture magnitude -> add empty columns\n", "for mag in [\"MMT u\", \"MMT g\", \"MMT i\", \"MMT z\", \"WIRCS J\", \"OMEGA2000 J\", \"OMEGA2000 Ks\"]:\n", " basecol = mag.replace(\" \", \"_\").lower()\n", " m_ap_col, merr_ap_col = \"m_ap_{}\".format(basecol), \"merr_ap_{}\".format(basecol)\n", " \n", " master_catalogue[m_ap_col] = [np.nan] * len(master_catalogue)\n", " master_catalogue[merr_ap_col] = [np.nan] * len(master_catalogue)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 1. Magnitudes and magnitude erros" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "collapsed": true }, "outputs": [], "source": [ "def flag_mag(flagcol, mask):\n", " \n", " # Add flag columns if does not exist\n", " if flagcol not in master_catalogue.colnames:\n", " master_catalogue[flagcol] = np.zeros(len(master_catalogue), dtype=bool)\n", " \n", " # Flagged\n", " master_catalogue[flagcol][mask] = np.ones(len(mask), dtype=bool)\n", " print(' Number of flagged objects:', len(master_catalogue[flagcol][mask]))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 1.a Pan-STARRS Aperture and Total magnitude errors" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "GPC1 g\n", " Aperture magnitude\n", " Number of flagged objects: 34\n", " Total magnitude\n", " Number of flagged objects: 37\n", "GPC1 r\n", " Aperture magnitude\n", " Number of flagged objects: 56\n", " Total magnitude\n", " Number of flagged objects: 55\n", "GPC1 i\n", " Aperture magnitude\n", " Number of flagged objects: 53\n", " Total magnitude\n", " Number of flagged objects: 58\n", "GPC1 z\n", " Aperture magnitude\n", " Number of flagged objects: 37\n", " Total magnitude\n", " Number of flagged objects: 34\n", "GPC1 y\n", " Aperture magnitude\n", " Number of flagged objects: 16\n", " Total magnitude\n", " Number of flagged objects: 18\n" ] } ], "source": [ "## dmu0: Pan-STARRS forced photometry cat \n", "gpc1_err = 0.0010860000038519502\n", "bands = [\"GPC1 g\", \"GPC1 r\", \"GPC1 i\", \"GPC1 z\", \"GPC1 y\"]\n", "\n", "for i, band in enumerate(bands):\n", " print(band)\n", " basecol = band.replace(\" \", \"_\").lower()\n", " \n", " ecol_ap, ecol_tot = \"merr_ap_{}\".format(basecol), \"merr_{}\".format(basecol)\n", " flagcol_ap, flagcol_tot = \"flag_ap_{}\".format(basecol), \"flag_{}\".format(basecol)\n", " \n", " mask_ap = np.where(master_catalogue[ecol_ap] == gpc1_err)[0]\n", " mask_tot = np.where(master_catalogue[ecol_tot] == gpc1_err)[0]\n", " \n", " print(' Aperture magnitude')\n", " flag_mag(flagcol_ap, mask_ap)\n", " print(' Total magnitude')\n", " flag_mag(flagcol_tot, mask_tot)\n", " " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 2.d WIRCS Aperture and Total K-band magnitudes" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "WIRCS K\n", " Aperture magnitude\n", " Number of flagged objects: 26\n", " Total magnitude\n", " Number of flagged objects: 58\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in greater_equal\n", " return getattr(self.data, op)(other)\n" ] } ], "source": [ "wircs_mag = 100.0\n", "bands = [\"WIRCS K\"]\n", "\n", "for i, band in enumerate(bands):\n", " print(band)\n", " basecol = band.replace(\" \", \"_\").lower()\n", " \n", " col_ap, col_tot = \"m_ap_{}\".format(basecol), \"m_{}\".format(basecol)\n", " flagcol_ap, flagcol_tot = \"flag_ap_{}\".format(basecol), \"flag_{}\".format(basecol)\n", " \n", " mask_ap = np.where(master_catalogue[col_ap] >= wircs_mag)[0]\n", " mask_tot = np.where(master_catalogue[col_tot] >= wircs_mag)[0]\n", " \n", " print(' Aperture magnitude')\n", " flag_mag(flagcol_ap, mask_ap)\n", " print(' Total magnitude')\n", " flag_mag(flagcol_tot, mask_tot)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 2. Outliers" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "No sources have both Megacam u (aperture) and MMT u (aperture) values.\n", "\n", "MMT u (total) - Megacam u (total):\n", " Number of outliers: 0\n", "SUPRIME g (aperture) - Megacam g (aperture):\n", " Number of outliers: 4\n", "SUPRIME g (total) - Megacam g (total):\n", " Number of outliers: 4\n", "GPC1 g (aperture) - Megacam g (aperture):\n", " Number of outliers: 0\n", "GPC1 g (total) - Megacam g (total):\n", " Number of outliers: 2\n", "90Prime g (aperture) - Megacam g (aperture):\n", " Number of outliers: 387\n", "90Prime g (total) - Megacam g (total):\n", " Number of outliers: 97\n", "No sources have both Megacam g (aperture) and MMT g (aperture) values.\n", "\n", "MMT g (total) - Megacam g (total):\n", " Number of outliers: 495\n", "GPC1 g (aperture) - SUPRIME g (aperture):\n", " Number of outliers: 0\n", "GPC1 g (total) - SUPRIME g (total):\n", " Number of outliers: 0\n", "90Prime g (aperture) - SUPRIME g (aperture):\n", " Number of outliers: 0\n", "90Prime g (total) - SUPRIME g (total):\n", " Number of outliers: 3\n", "No sources have both SUPRIME g (aperture) and MMT g (aperture) values.\n", "\n", "MMT g (total) - SUPRIME g (total):\n", " Number of outliers: 1\n", "90Prime g (aperture) - GPC1 g (aperture):\n", " Number of outliers: 19\n", "90Prime g (total) - GPC1 g (total):\n", " Number of outliers: 6\n", "No sources have both GPC1 g (aperture) and MMT g (aperture) values.\n", "\n", "MMT g (total) - GPC1 g (total):\n", " Number of outliers: 0\n", "No sources have both 90Prime g (aperture) and MMT g (aperture) values.\n", "\n", "MMT g (total) - 90Prime g (total):\n", " Number of outliers: 2\n", "SUPRIME r (aperture) - Megacam r (aperture):\n", " Number of outliers: 5\n", "SUPRIME r (total) - Megacam r (total):\n", " Number of outliers: 4\n", "GPC1 r (aperture) - Megacam r (aperture):\n", " Number of outliers: 0\n", "GPC1 r (total) - Megacam r (total):\n", " Number of outliers: 4\n", "90Prime r (aperture) - Megacam r (aperture):\n", " Number of outliers: 0\n", "90Prime r (total) - Megacam r (total):\n", " Number of outliers: 6\n", "GPC1 r (aperture) - SUPRIME r (aperture):\n", " Number of outliers: 0\n", "GPC1 r (total) - SUPRIME r (total):\n", " Number of outliers: 0\n", "90Prime r (aperture) - SUPRIME r (aperture):\n", " Number of outliers: 0\n", "90Prime r (total) - SUPRIME r (total):\n", " Number of outliers: 5\n", "90Prime r (aperture) - GPC1 r (aperture):\n", " Number of outliers: 504\n", "90Prime r (total) - GPC1 r (total):\n", " Number of outliers: 0\n", "SUPRIME i (aperture) - Megacam i (aperture):\n", " Number of outliers: 28\n", "SUPRIME i (total) - Megacam i (total):\n", " Number of outliers: 5\n", "GPC1 i (aperture) - Megacam i (aperture):\n", " Number of outliers: 0\n", "GPC1 i (total) - Megacam i (total):\n", " Number of outliers: 4\n", "No sources have both Megacam i (aperture) and MMT i (aperture) values.\n", "\n", "MMT i (total) - Megacam i (total):\n", " Number of outliers: 20\n", "GPC1 i (aperture) - SUPRIME i (aperture):\n", " Number of outliers: 0\n", "GPC1 i (total) - SUPRIME i (total):\n", " Number of outliers: 10\n", "No sources have both SUPRIME i (aperture) and MMT i (aperture) values.\n", "\n", "MMT i (total) - SUPRIME i (total):\n", " Number of outliers: 2\n", "No sources have both GPC1 i (aperture) and MMT i (aperture) values.\n", "\n", "MMT i (total) - GPC1 i (total):\n", " Number of outliers: 2\n", "SUPRIME z (aperture) - Megacam z (aperture):\n", " Number of outliers: 3\n", "SUPRIME z (total) - Megacam z (total):\n", " Number of outliers: 5\n", "GPC1 z (aperture) - Megacam z (aperture):\n", " Number of outliers: 0\n", "GPC1 z (total) - Megacam z (total):\n", " Number of outliers: 4\n", "Mosaic z (aperture) - Megacam z (aperture):\n", " Number of outliers: 0\n", "Mosaic z (total) - Megacam z (total):\n", " Number of outliers: 0\n", "No sources have both Megacam z (aperture) and MMT z (aperture) values.\n", "\n", "MMT z (total) - Megacam z (total):\n", " Number of outliers: 26\n", "GPC1 z (aperture) - SUPRIME z (aperture):\n", " Number of outliers: 0\n", "GPC1 z (total) - SUPRIME z (total):\n", " Number of outliers: 1\n", "Mosaic z (aperture) - SUPRIME z (aperture):\n", " Number of outliers: 0\n", "Mosaic z (total) - SUPRIME z (total):\n", " Number of outliers: 0\n", "No sources have both SUPRIME z (aperture) and MMT z (aperture) values.\n", "\n", "MMT z (total) - SUPRIME z (total):\n", " Number of outliers: 0\n", "Mosaic z (aperture) - GPC1 z (aperture):\n", " Number of outliers: 531\n", "Mosaic z (total) - GPC1 z (total):\n", " Number of outliers: 0\n", "No sources have both GPC1 z (aperture) and MMT z (aperture) values.\n", "\n", "MMT z (total) - GPC1 z (total):\n", " Number of outliers: 2\n", "No sources have both Mosaic z (aperture) and MMT z (aperture) values.\n", "\n", "MMT z (total) - Mosaic z (total):\n", " Number of outliers: 0\n", "GPC1 y (aperture) - SUPRIME y (aperture):\n", " Number of outliers: 0\n", "GPC1 y (total) - SUPRIME y (total):\n", " Number of outliers: 0\n", "No sources have both WIRCAM J (aperture) and WIRCS J (aperture) values.\n", "\n", "WIRCS J (total) - WIRCAM J (total):\n", " Number of outliers: 2\n", "UKIDSS J (aperture) - WIRCAM J (aperture):\n", " Number of outliers: 13\n", "UKIDSS J (total) - WIRCAM J (total):\n", " Number of outliers: 71\n", "No sources have both WIRCAM J (aperture) and OMEGA2000 J (aperture) values.\n", "\n", "OMEGA2000 J (total) - WIRCAM J (total):\n", " Number of outliers: 1\n", "No sources have both WIRCS J (aperture) and UKIDSS J (aperture) values.\n", "\n", "UKIDSS J (total) - WIRCS J (total):\n", " Number of outliers: 1\n", "No sources have both WIRCS J (aperture) and OMEGA2000 J (aperture) values.\n", "\n", "OMEGA2000 J (total) - WIRCS J (total):\n", " Number of outliers: 0\n", "No sources have both UKIDSS J (aperture) and OMEGA2000 J (aperture) values.\n", "\n", "OMEGA2000 J (total) - UKIDSS J (total):\n", " Number of outliers: 2\n", "WIRCS K (aperture) - WIRCAM Ks (aperture):\n", " Number of outliers: 21\n", "WIRCS K (total) - WIRCAM Ks (total):\n", " Number of outliers: 7\n", "No sources have both WIRCAM Ks (aperture) and NEWFIRM K (aperture) values.\n", "\n", "NEWFIRM K (total) - WIRCAM Ks (total):\n", " Number of outliers: 0\n", "No sources have both WIRCAM Ks (aperture) and OMEGA2000 Ks (aperture) values.\n", "\n", "OMEGA2000 Ks (total) - WIRCAM Ks (total):\n", " Number of outliers: 0\n", "No sources have both WIRCS K (aperture) and NEWFIRM K (aperture) values.\n", "\n", "NEWFIRM K (total) - WIRCS K (total):\n", " Number of outliers: 0\n", "No sources have both WIRCS K (aperture) and OMEGA2000 Ks (aperture) values.\n", "\n", "OMEGA2000 Ks (total) - WIRCS K (total):\n", " Number of outliers: 0\n", "No sources have both NEWFIRM K (aperture) and OMEGA2000 Ks (aperture) values.\n", "\n", "OMEGA2000 Ks (total) - NEWFIRM K (total):\n", " Number of outliers: 0\n" ] } ], "source": [ "for band_of_a_kind in all_bands:\n", " for band1, band2 in itertools.combinations(band_of_a_kind, 2):\n", " #print(band1, band2)\n", "\n", " basecol1, basecol2 = band1.replace(\" \", \"_\").lower(), band2.replace(\" \", \"_\").lower()\n", " \n", " # Aperture mag\n", " col1, col2 = \"m_ap_{}\".format(basecol1), \"m_ap_{}\".format(basecol2)\n", " ecol1, ecol2 = \"merr_ap_{}\".format(basecol1), \"merr_ap_{}\".format(basecol2)\n", " flagcol1, flagcol2 = \"flag_ap_{}\".format(basecol1), \"flag_ap_{}\".format(basecol2)\n", " \n", " try:\n", " master_catalogue = flag_outliers(master_catalogue, col1, col2,\n", " ecol1, ecol2,\n", " flagcol1, flagcol2,\n", " labels=(\"{} (aperture)\".format(band1), \"{} (aperture)\".format(band2)))\n", " except KeyError:\n", " print(\"One of {} and {} is not present in the catalogue.\".format(col1, col2))\n", " \n", " \n", " # Tot mag\n", " col1, col2 = \"m_{}\".format(basecol1), \"m_{}\".format(basecol2) \n", " ecol1, ecol2 = \"merr_{}\".format(basecol1), \"merr_{}\".format(basecol2) \n", " flagcol1, flagcol2 = \"flag_{}\".format(basecol1), \"flag_{}\".format(basecol2)\n", " \n", " try:\n", " master_catalogue = flag_outliers(master_catalogue, col1, col2, \n", " ecol1, ecol2,\n", " flagcol1, flagcol2,\n", " labels=(\"{} (total)\".format(band1), \"{} (total)\".format(band2))) \n", " except KeyError:\n", " print(\"One of {} and {} is not present in the catalogue.\".format(col1, col2))\n", " " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 3. Save table" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "collapsed": true }, "outputs": [], "source": [ "\n", "#Merge any aperture flags#Merge \n", "for col in master_catalogue.colnames:\n", " if col.startswith(\"flag_ap_\"):\n", " try:\n", " master_catalogue[col.replace(\"_ap_\", \"_\")] = (master_catalogue[col.replace(\"_ap_\", \"_\")] |\n", " master_catalogue[col])\n", " master_catalogue.remove_column(col)\n", " except KeyError:\n", " print(\"{} only has aperture flags.\".format(col))\n", " master_catalogue.rename_column(col, col.replace(\"_ap_\", \"_\"))" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "collapsed": true }, "outputs": [], "source": [ "flag_cols = [\"help_id\"]\n", "for col in master_catalogue.colnames:\n", " if col.startswith(\"flag_\"):\n", " flag_cols += [col]\n", "new_catname = \"./data/{}_{}_flags.fits\".format(FIELD.lower(),SUFFIX)\n", "master_catalogue[flag_cols].write(new_catname, overwrite = True)" ] } ], "metadata": { "kernelspec": { "display_name": "Python (herschelhelp_internal)", "language": "python", "name": "helpint" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.4" } }, "nbformat": 4, "nbformat_minor": 1 }