{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This notebook was run with herschelhelp_internal version: \n",
      "017bb1e (Mon Jun 18 14:58:59 2018 +0100) [with local modifications]\n",
      "This notebook was executed on: \n",
      "2021-05-18 12:36:48.599808\n"
     ]
    }
   ],
   "source": [
    "from herschelhelp_internal import git_version\n",
    "print(\"This notebook was run with herschelhelp_internal version: \\n{}\".format(git_version()))\n",
    "import datetime\n",
    "print(\"This notebook was executed on: \\n{}\".format(datetime.datetime.now()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/matplotlib/__init__.py:855: MatplotlibDeprecationWarning: \n",
      "examples.directory is deprecated; in the future, examples will be found relative to the 'datapath' directory.\n",
      "  \"found relative to the 'datapath' directory.\".format(key))\n",
      "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/matplotlib/__init__.py:846: MatplotlibDeprecationWarning: \n",
      "The text.latex.unicode rcparam was deprecated in Matplotlib 2.2 and will be removed in 3.1.\n",
      "  \"2.2\", name=key, obj_type=\"rcparam\", addendum=addendum)\n",
      "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/seaborn/apionly.py:9: UserWarning: As seaborn no longer sets a default style on import, the seaborn.apionly module is deprecated. It will be removed in a future version.\n",
      "  warnings.warn(msg, UserWarning)\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "%matplotlib inline\n",
    "#%config InlineBackend.figure_format = 'svg'\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rc('figure', figsize=(10, 6))\n",
    "\n",
    "import os\n",
    "import time\n",
    "import gc\n",
    "import glob\n",
    "from matplotlib_venn import venn2\n",
    "from astropy import units as u\n",
    "from astropy.coordinates import SkyCoord\n",
    "from astropy.table import Column, QTable, Table, join, vstack, hstack, setdiff, unique\n",
    "import numpy as np\n",
    "import pymoc \n",
    "from pymoc import MOC\n",
    "from datetime import datetime\n",
    "\n",
    "\n",
    "from herschelhelp_internal.masterlist import merge_catalogues, nb_merge_dist_plot, specz_merge\n",
    "from herschelhelp_internal.utils import coords_to_hpidx, ebv, gen_help_id, inMoc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "DMU_DIR = '../'\n",
    "FIELD = 'Herschel-Stripe-82'\n",
    "SUFFIX = '20210513' #datetime.today().strftime('%Y%m%d')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "55"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def select_sample(catalogue, allbands, mocs_list, mcuts, ndet=2):\n",
    "    \"\"\"\n",
    "    param.\n",
    "    ---------------\n",
    "    catalogue: astropy.table\n",
    "        HELP catalogue \n",
    "    allbands: [[..., str]]\n",
    "        Names of bands used in the selections: [[opt_bands], [nir_bands], [mir_bands]]\n",
    "    mocs_list: : [..., str] \n",
    "        list containing the mocs fits files.\n",
    "    mcuts: dict\n",
    "        magnitude cuts limits for each band.\n",
    "    ndet: int\n",
    "        number of detections required per group of bands (opt/nir/mid). Default 2; it requires 2 detection on each to pass the selection.\n",
    "    \n",
    "    output\n",
    "    ---------------\n",
    "    nb_band: astropy.Table \n",
    "        Table with flags for each source indicating whether the source has at least \"ndet\" detections above the magnitude cut.\n",
    "        Flags: optband=1, nirband=2, mirband=4, ndet_total = (optband + nirband + mirband)\n",
    "    \"\"\"\n",
    "    import numpy as np\n",
    "    from astropy.table import Column, Table, join\n",
    "    import time\n",
    "    from pymoc import MOC\n",
    "    \n",
    "    \n",
    "    nb_band = {}\n",
    "    nb_band['help_id'] = catalogue['help_id']\n",
    "    nb = ['optband', 'nirband', 'mirband']\n",
    "    nx = ['optvalue', 'nirvalue', 'mirvalue']\n",
    "    \n",
    "    r_filter = np.zeros(len(catalogue))\n",
    "\n",
    "    # Name of all moc files \n",
    "    mocs = [x[12:] for x in mocs_list]\n",
    "    path_mocs = mocs_list[0][:12]\n",
    "                  \n",
    "\n",
    "    for idx, bands in enumerate(allbands):\n",
    "        nb_band[nb[idx]] = np.zeros(len(catalogue))\n",
    "        nb_filter = np.zeros(len(catalogue))\n",
    "\n",
    "        for i, filt in enumerate(bands):\n",
    "            # mocs in that filter\n",
    "            mocs_filt = [moc for moc in mocs if filt in moc]\n",
    "            ind = np.zeros(len(catalogue))\n",
    "            for moc_ in mocs_filt:\n",
    "                # Get the name of the filter and survey\n",
    "                m = moc_[:-18]\n",
    "                \n",
    "                # Read in the moc for the specific survey     \n",
    "                moc = MOC()\n",
    "                moc.read(path_mocs+moc_)\n",
    "                ind_moc = inMoc(catalogue['ra'], catalogue['dec'],moc)\n",
    "                \n",
    "                # Source 'detection'\n",
    "                ind_filter = 1 * (catalogue['m_'+filt] <= mcuts[m]) # Sources with magnitude below the magnitud cut defined for that survey\n",
    "                ind_filter = ind_filter * ind_moc # Sources within the moc survey\n",
    "                ind += ind_filter\n",
    "\n",
    "            nb_filter += ind \n",
    "            # Count sources detected on the 'r' filter\n",
    "            if '_r' in filt:\n",
    "                r_filter += ind                   \n",
    "                \n",
    "        # sources detected in at least two filters of those bands (opt, nir, mir)\n",
    "        if idx==2:\n",
    "            n = 2\n",
    "        else:\n",
    "            n = 1\n",
    "        # add flag: 1 * opt ; 2 * nir ; 4 * mir    \n",
    "        nb_band[nb[idx]] += (idx+n) * (nb_filter >= ndet)  \n",
    "    \n",
    "    nb_band = Table(nb_band)\n",
    "    \n",
    "    # at least 1det in r band\n",
    "    has_r_det = 1 * (r_filter >= 1)\n",
    "    nb_band['optband'] = nb_band['optband'] * has_r_det\n",
    "            \n",
    "    total = np.zeros(len(catalogue))\n",
    "    for i in range(len(allbands)):\n",
    "        total += nb_band[nb[i]]\n",
    "\n",
    "    nb_band['ndet_total'] = total\n",
    "    \n",
    "    return nb_band\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def select_sample_subfiles(file_list, allbands, mocs_list, mcut, ndet=2):\n",
    "    \n",
    "    nb = ['optband', 'nirband', 'mirband']\n",
    "    names_det = ['help_id'] + nb[:len(allbands)] + ['ndet_total']\n",
    "    nb_band = Table(names=names_det, dtype=('str','float','float','float'))  \n",
    "    \n",
    "    # read sub_files to calculate snr per each band\n",
    "    ################################################\n",
    "    print('Reading sub_catalogues')\n",
    "    start = time.time()  \n",
    "    for idx, sub_file in enumerate(file_list):\n",
    "        print('subfile: ', idx)\n",
    "        start_sub = time.time() \n",
    "        \n",
    "        sub_catalogue = Table.read(sub_file)\n",
    "        sub_nb_band = select_sample(sub_catalogue, allbands, mocs_list, mcut, ndet=ndet)\n",
    "        nb_band = vstack([nb_band, sub_nb_band])\n",
    "        \n",
    "        end_sub = time.time()\n",
    "        time_ = end_sub - start_sub\n",
    "        print('select_sample subcatalogue time: ', time_)\n",
    "        \n",
    "        del sub_catalogue\n",
    "        del sub_nb_band\n",
    "        gc.collect()\n",
    "    \n",
    "    end = time.time()\n",
    "    time_all = end - start\n",
    "    print('select_sample total time: ', time_all)    \n",
    "    \n",
    "    return nb_band     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compare_cats_venndiagram(cat1, cat2, ind_catalogue, ind_names, cat1name, cat2name, save=True):\n",
    "    \"\"\"\n",
    "    \"\"\"\n",
    "    for i, ind_cat in enumerate(ind_catalogue):\n",
    "#             print('joining catalogues')\n",
    "        plt.figure(figsize=(10,10))\n",
    "        set1 = set(cat1.tolist())\n",
    "        set2 = set(cat2[ind_cat].tolist())\n",
    "\n",
    "        out = venn2([set1, set2], (cat1name, cat2name))\n",
    "\n",
    "        for text in out.set_labels:\n",
    "            text.set_fontsize(20)\n",
    "        plt.title(ind_names[i])\n",
    "        \n",
    "        if save==True:\n",
    "            plt.savefig('./data/figs/'+ind_names[i]+'.png', format='png')\n",
    "            plt.savefig('./data/figs/'+ind_names[i]+'.pdf', format='pdf')\n",
    "        plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. SELECTION FUNCTION\n",
    "\n",
    "### 1.1 Read in files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### MOCs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "28"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "moc_list = glob.glob('./mocs/*.fits')\n",
    "len(moc_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Masterlist / Subfiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 502 subfiles\n"
     ]
    }
   ],
   "source": [
    "use_subfiles = True\n",
    "\n",
    "if use_subfiles == False:\n",
    "    masterfile = DMU_DIR + 'dmu1/dmu1_ml_Herschel-Stripe-82/data/master_catalogue_herschel-stripe-82_{}.fits'.format(SUFFIX) # too big\n",
    "    masterlist = Table.read(masterfile, memmap=True)\n",
    "\n",
    "else:\n",
    "    file_list = glob.glob(DMU_DIR + 'dmu1/dmu1_ml_Herschel-Stripe-82/data/tiles/sub_catalogue_herschel-stripe-82_{}_*.fits'.format(SUFFIX))\n",
    "    print('There are {} subfiles'.format(len(file_list)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.2 Bands definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# HELP bands\n",
    "############\n",
    "\n",
    "u = [                        'sdss_u']\n",
    "g = ['suprime_g', 'decam_g', 'sdss_g']\n",
    "r = ['suprime_r', 'decam_r', 'sdss_r']\n",
    "i = ['suprime_i', 'decam_i', 'sdss_i']\n",
    "z = ['suprime_z', 'decam_z', 'sdss_z']\n",
    "y = ['suprime_y', 'decam_y']\n",
    "\n",
    "y_nir = ['vista_y',  'ukidss_y']\n",
    "h = ['vista_h',  'ukidss_h']\n",
    "j = ['vista_j',  'ukidss_j']\n",
    "k = ['vista_ks', 'ukidss_k']\n",
    "\n",
    "helpbands = [u + g + r + i + z + y] + [y_nir + h + j + k] "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.3 Magnitude cuts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SDSS\n",
    "magcuts_sdss = {'sdss_u': 22.8,\n",
    " 'sdss_g': 23.7,\n",
    " 'sdss_r': 23.5,\n",
    " 'sdss_i': 22.9,\n",
    " 'sdss_z': 21.5}\n",
    "\n",
    "# HSC\n",
    "magcuts_hsc = {\n",
    "    'suprime_g': 25.4,\n",
    "    'suprime_r': 25.4, \n",
    "    'suprime_i': 25.1,\n",
    "    'suprime_z': 24.2,\n",
    "    'suprime_y': 23.7}\n",
    "\n",
    "# DECam: DECaLS + DES\n",
    "magcuts_des = {'decam_g_des': 23.3,\n",
    " 'decam_r_des': 23.1,\n",
    " 'decam_i_des': 22.7,\n",
    " 'decam_z_des': 21.8,\n",
    " 'decam_y_des': 20.6}\n",
    "\n",
    "magcuts_decals =  {'decam_g_decals': 23.9, 'decam_r_decals': 23.5, 'decam_z_decals': 22.5}\n",
    "\n",
    "# VISTA: VHS + VICS82\n",
    "magcuts_vhs = {'vista_y_vhs': 20.1,\n",
    " 'vista_h_vhs': 19.6,\n",
    " 'vista_j_vhs': 19.9,\n",
    " 'vista_ks_vhs': 19.6}\n",
    "\n",
    "magcuts_vics = {'vista_j_vics': 21.0,\n",
    " 'vista_ks_vics': 21.0}\n",
    "\n",
    "# UKIDSS - LAS\n",
    "magcuts_ukidss = {'ukidss_h':19.0, 'ukidss_j':19.0, 'ukidss_k':19.0, 'ukidss_y':20.2}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'sdss_u': 22.8,\n",
       " 'sdss_g': 23.7,\n",
       " 'sdss_r': 23.5,\n",
       " 'sdss_i': 22.9,\n",
       " 'sdss_z': 21.5,\n",
       " 'suprime_g': 25.4,\n",
       " 'suprime_r': 25.4,\n",
       " 'suprime_i': 25.1,\n",
       " 'suprime_z': 24.2,\n",
       " 'suprime_y': 23.7,\n",
       " 'decam_g_des': 23.3,\n",
       " 'decam_r_des': 23.1,\n",
       " 'decam_i_des': 22.7,\n",
       " 'decam_z_des': 21.8,\n",
       " 'decam_y_des': 20.6,\n",
       " 'decam_g_decals': 23.9,\n",
       " 'decam_r_decals': 23.5,\n",
       " 'decam_z_decals': 22.5,\n",
       " 'vista_y_vhs': 20.1,\n",
       " 'vista_h_vhs': 19.6,\n",
       " 'vista_j_vhs': 19.9,\n",
       " 'vista_ks_vhs': 19.6,\n",
       " 'vista_j_vics': 21.0,\n",
       " 'vista_ks_vics': 21.0,\n",
       " 'ukidss_h': 19.0,\n",
       " 'ukidss_j': 19.0,\n",
       " 'ukidss_k': 19.0,\n",
       " 'ukidss_y': 20.2}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mcuts = magcuts_sdss.copy()\n",
    "\n",
    "mcuts.update(magcuts_hsc)\n",
    "mcuts.update(magcuts_des)\n",
    "mcuts.update(magcuts_decals)\n",
    "mcuts.update(magcuts_vhs)\n",
    "mcuts.update(magcuts_vics)\n",
    "mcuts.update(magcuts_ukidss)\n",
    "mcuts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.4 Flagged catalogue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading sub_catalogues\n",
      "subfile:  0\n",
      "select_sample subcatalogue time:  0.49558401107788086\n",
      "subfile:  1\n",
      "select_sample subcatalogue time:  0.5206484794616699\n",
      "subfile:  2\n",
      "select_sample subcatalogue time:  0.5457777976989746\n",
      "subfile:  3\n",
      "select_sample subcatalogue time:  0.5766429901123047\n",
      "subfile:  4\n",
      "select_sample subcatalogue time:  0.6113932132720947\n",
      "subfile:  5\n",
      "select_sample subcatalogue time:  0.6475973129272461\n",
      "subfile:  6\n",
      "select_sample subcatalogue time:  0.6827290058135986\n",
      "subfile:  7\n",
      "select_sample subcatalogue time:  0.7350175380706787\n",
      "subfile:  8\n",
      "select_sample subcatalogue time:  0.7561945915222168\n",
      "subfile:  9\n",
      "select_sample subcatalogue time:  0.7928931713104248\n",
      "subfile:  10\n",
      "select_sample subcatalogue time:  0.8286569118499756\n",
      "subfile:  11\n",
      "select_sample subcatalogue time:  0.8662714958190918\n",
      "subfile:  12\n",
      "select_sample subcatalogue time:  0.9220988750457764\n",
      "subfile:  13\n",
      "select_sample subcatalogue time:  0.9803392887115479\n",
      "subfile:  14\n",
      "select_sample subcatalogue time:  0.980527400970459\n",
      "subfile:  15\n",
      "select_sample subcatalogue time:  1.0142977237701416\n",
      "subfile:  16\n",
      "select_sample subcatalogue time:  1.0535478591918945\n",
      "subfile:  17\n",
      "select_sample subcatalogue time:  1.087165117263794\n",
      "subfile:  18\n",
      "select_sample subcatalogue time:  1.1254949569702148\n",
      "subfile:  19\n",
      "select_sample subcatalogue time:  1.1817386150360107\n",
      "subfile:  20\n",
      "select_sample subcatalogue time:  1.2022266387939453\n",
      "subfile:  21\n",
      "select_sample subcatalogue time:  1.232548713684082\n",
      "subfile:  22\n",
      "select_sample subcatalogue time:  1.2965192794799805\n",
      "subfile:  23\n",
      "select_sample subcatalogue time:  1.3151607513427734\n",
      "subfile:  24\n",
      "select_sample subcatalogue time:  1.3472223281860352\n",
      "subfile:  25\n",
      "select_sample subcatalogue time:  1.3698740005493164\n",
      "subfile:  26\n",
      "select_sample subcatalogue time:  1.431135892868042\n",
      "subfile:  27\n",
      "select_sample subcatalogue time:  1.440422534942627\n",
      "subfile:  28\n",
      "select_sample subcatalogue time:  1.5292994976043701\n",
      "subfile:  29\n",
      "select_sample subcatalogue time:  1.5156662464141846\n",
      "subfile:  30\n",
      "select_sample subcatalogue time:  1.581800937652588\n",
      "subfile:  31\n",
      "select_sample subcatalogue time:  1.5895061492919922\n",
      "subfile:  32\n",
      "select_sample subcatalogue time:  1.6557834148406982\n",
      "subfile:  33\n",
      "select_sample subcatalogue time:  1.6592075824737549\n",
      "subfile:  34\n",
      "select_sample subcatalogue time:  1.7270655632019043\n",
      "subfile:  35\n",
      "select_sample subcatalogue time:  1.7285072803497314\n",
      "subfile:  36\n",
      "select_sample subcatalogue time:  1.8316576480865479\n",
      "subfile:  37\n",
      "select_sample subcatalogue time:  1.8057034015655518\n",
      "subfile:  38\n",
      "select_sample subcatalogue time:  1.8849999904632568\n",
      "subfile:  39\n",
      "select_sample subcatalogue time:  1.880246877670288\n",
      "subfile:  40\n",
      "select_sample subcatalogue time:  1.947326421737671\n",
      "subfile:  41\n",
      "select_sample subcatalogue time:  1.9462361335754395\n",
      "subfile:  42\n",
      "select_sample subcatalogue time:  2.040663242340088\n",
      "subfile:  43\n",
      "select_sample subcatalogue time:  2.0573999881744385\n",
      "subfile:  44\n",
      "select_sample subcatalogue time:  2.084383487701416\n",
      "subfile:  45\n",
      "select_sample subcatalogue time:  2.1168758869171143\n",
      "subfile:  46\n",
      "select_sample subcatalogue time:  2.1513001918792725\n",
      "subfile:  47\n",
      "select_sample subcatalogue time:  2.1894564628601074\n",
      "subfile:  48\n",
      "select_sample subcatalogue time:  2.2263591289520264\n",
      "subfile:  49\n",
      "select_sample subcatalogue time:  2.272230863571167\n",
      "subfile:  50\n",
      "select_sample subcatalogue time:  2.3033363819122314\n",
      "subfile:  51\n",
      "select_sample subcatalogue time:  2.3338446617126465\n",
      "subfile:  52\n",
      "select_sample subcatalogue time:  2.369276762008667\n",
      "subfile:  53\n",
      "select_sample subcatalogue time:  2.408170223236084\n",
      "subfile:  54\n",
      "select_sample subcatalogue time:  2.4497389793395996\n",
      "subfile:  55\n",
      "select_sample subcatalogue time:  2.478421211242676\n",
      "subfile:  56\n",
      "select_sample subcatalogue time:  2.5193381309509277\n",
      "subfile:  57\n",
      "select_sample subcatalogue time:  2.5552124977111816\n",
      "subfile:  58\n",
      "select_sample subcatalogue time:  2.5953078269958496\n",
      "subfile:  59\n",
      "select_sample subcatalogue time:  2.631927490234375\n",
      "subfile:  60\n",
      "select_sample subcatalogue time:  2.665588140487671\n",
      "subfile:  61\n",
      "select_sample subcatalogue time:  2.713979959487915\n",
      "subfile:  62\n",
      "select_sample subcatalogue time:  2.736506462097168\n",
      "subfile:  63\n",
      "select_sample subcatalogue time:  2.775275945663452\n",
      "subfile:  64\n",
      "select_sample subcatalogue time:  2.809053421020508\n",
      "subfile:  65\n",
      "select_sample subcatalogue time:  2.8444857597351074\n",
      "subfile:  66\n",
      "select_sample subcatalogue time:  2.8853394985198975\n",
      "subfile:  67\n",
      "select_sample subcatalogue time:  2.922278881072998\n",
      "subfile:  68\n",
      "select_sample subcatalogue time:  2.9571750164031982\n",
      "subfile:  69\n",
      "select_sample subcatalogue time:  2.9928059577941895\n",
      "subfile:  70\n",
      "select_sample subcatalogue time:  3.0335073471069336\n",
      "subfile:  71\n",
      "select_sample subcatalogue time:  3.072092056274414\n",
      "subfile:  72\n",
      "select_sample subcatalogue time:  3.1090774536132812\n",
      "subfile:  73\n",
      "select_sample subcatalogue time:  3.142198085784912\n",
      "subfile:  74\n",
      "select_sample subcatalogue time:  3.183154582977295\n",
      "subfile:  75\n",
      "select_sample subcatalogue time:  3.210667848587036\n",
      "subfile:  76\n",
      "select_sample subcatalogue time:  3.249861001968384\n",
      "subfile:  77\n",
      "select_sample subcatalogue time:  3.286318063735962\n",
      "subfile:  78\n",
      "select_sample subcatalogue time:  3.327597141265869\n",
      "subfile:  79\n",
      "select_sample subcatalogue time:  3.3569982051849365\n",
      "subfile:  80\n",
      "select_sample subcatalogue time:  3.3982224464416504\n",
      "subfile:  81\n",
      "select_sample subcatalogue time:  3.4343361854553223\n",
      "subfile:  82\n",
      "select_sample subcatalogue time:  3.4714527130126953\n",
      "subfile:  83\n",
      "select_sample subcatalogue time:  3.503584146499634\n",
      "subfile:  84\n",
      "select_sample subcatalogue time:  3.5456748008728027\n",
      "subfile:  85\n",
      "select_sample subcatalogue time:  3.576779365539551\n",
      "subfile:  86\n",
      "select_sample subcatalogue time:  3.6124494075775146\n",
      "subfile:  87\n",
      "select_sample subcatalogue time:  3.6485674381256104\n",
      "subfile:  88\n",
      "select_sample subcatalogue time:  3.7021968364715576\n",
      "subfile:  89\n",
      "select_sample subcatalogue time:  3.7239878177642822\n",
      "subfile:  90\n",
      "select_sample subcatalogue time:  3.760159492492676\n",
      "subfile:  91\n",
      "select_sample subcatalogue time:  3.801299810409546\n",
      "subfile:  92\n",
      "select_sample subcatalogue time:  3.8372952938079834\n",
      "subfile:  93\n",
      "select_sample subcatalogue time:  3.8693339824676514\n",
      "subfile:  94\n",
      "select_sample subcatalogue time:  3.911472797393799\n",
      "subfile:  95\n",
      "select_sample subcatalogue time:  3.9639227390289307\n",
      "subfile:  96\n",
      "select_sample subcatalogue time:  3.9795541763305664\n",
      "subfile:  97\n",
      "select_sample subcatalogue time:  4.021539926528931\n",
      "subfile:  98\n",
      "select_sample subcatalogue time:  4.053730249404907\n",
      "subfile:  99\n",
      "select_sample subcatalogue time:  4.086378812789917\n",
      "subfile:  100\n",
      "select_sample subcatalogue time:  4.130409240722656\n",
      "subfile:  101\n",
      "select_sample subcatalogue time:  4.162182092666626\n",
      "subfile:  102\n",
      "select_sample subcatalogue time:  4.20279335975647\n",
      "subfile:  103\n",
      "select_sample subcatalogue time:  4.246006011962891\n",
      "subfile:  104\n",
      "select_sample subcatalogue time:  4.273890256881714\n",
      "subfile:  105\n",
      "select_sample subcatalogue time:  4.307199954986572\n",
      "subfile:  106\n",
      "select_sample subcatalogue time:  4.349309206008911\n",
      "subfile:  107\n",
      "select_sample subcatalogue time:  4.381016969680786\n",
      "subfile:  108\n",
      "select_sample subcatalogue time:  4.42132568359375\n",
      "subfile:  109\n",
      "select_sample subcatalogue time:  4.4528515338897705\n",
      "subfile:  110\n",
      "select_sample subcatalogue time:  4.490609407424927\n",
      "subfile:  111\n",
      "select_sample subcatalogue time:  4.5294036865234375\n",
      "subfile:  112\n",
      "select_sample subcatalogue time:  4.572713613510132\n",
      "subfile:  113\n",
      "select_sample subcatalogue time:  4.5963523387908936\n",
      "subfile:  114\n",
      "select_sample subcatalogue time:  4.645019292831421\n",
      "subfile:  115\n",
      "select_sample subcatalogue time:  4.674178123474121\n",
      "subfile:  116\n",
      "select_sample subcatalogue time:  4.7170569896698\n",
      "subfile:  117\n",
      "select_sample subcatalogue time:  4.742801904678345\n",
      "subfile:  118\n",
      "select_sample subcatalogue time:  4.786360502243042\n",
      "subfile:  119\n",
      "select_sample subcatalogue time:  4.821988821029663\n",
      "subfile:  120\n",
      "select_sample subcatalogue time:  4.857168912887573\n",
      "subfile:  121\n",
      "select_sample subcatalogue time:  4.895545482635498\n",
      "subfile:  122\n",
      "select_sample subcatalogue time:  4.933082580566406\n",
      "subfile:  123\n",
      "select_sample subcatalogue time:  4.9621741771698\n",
      "subfile:  124\n",
      "select_sample subcatalogue time:  5.007430076599121\n",
      "subfile:  125\n",
      "select_sample subcatalogue time:  5.036048412322998\n",
      "subfile:  126\n",
      "select_sample subcatalogue time:  5.0775065422058105\n",
      "subfile:  127\n",
      "select_sample subcatalogue time:  5.1117119789123535\n",
      "subfile:  128\n",
      "select_sample subcatalogue time:  5.151935338973999\n",
      "subfile:  129\n",
      "select_sample subcatalogue time:  5.187598943710327\n",
      "subfile:  130\n",
      "select_sample subcatalogue time:  5.218937873840332\n",
      "subfile:  131\n",
      "select_sample subcatalogue time:  5.271618604660034\n",
      "subfile:  132\n",
      "select_sample subcatalogue time:  5.307069301605225\n",
      "subfile:  133\n",
      "select_sample subcatalogue time:  5.330812454223633\n",
      "subfile:  134\n",
      "select_sample subcatalogue time:  5.36889123916626\n",
      "subfile:  135\n",
      "select_sample subcatalogue time:  5.4040327072143555\n",
      "subfile:  136\n",
      "select_sample subcatalogue time:  5.457367897033691\n",
      "subfile:  137\n",
      "select_sample subcatalogue time:  5.475980043411255\n",
      "subfile:  138\n",
      "select_sample subcatalogue time:  5.520779609680176\n",
      "subfile:  139\n",
      "select_sample subcatalogue time:  5.547765016555786\n",
      "subfile:  140\n",
      "select_sample subcatalogue time:  5.597963094711304\n",
      "subfile:  141\n",
      "select_sample subcatalogue time:  5.62513542175293\n",
      "subfile:  142\n",
      "select_sample subcatalogue time:  5.664100885391235\n",
      "subfile:  143\n",
      "select_sample subcatalogue time:  5.704246520996094\n",
      "subfile:  144\n",
      "select_sample subcatalogue time:  5.73552131652832\n",
      "subfile:  145\n",
      "select_sample subcatalogue time:  5.772716045379639\n",
      "subfile:  146\n",
      "select_sample subcatalogue time:  5.814227819442749\n",
      "subfile:  147\n",
      "select_sample subcatalogue time:  5.8498735427856445\n",
      "subfile:  148\n",
      "select_sample subcatalogue time:  5.886845588684082\n",
      "subfile:  149\n",
      "select_sample subcatalogue time:  5.913197040557861\n",
      "subfile:  150\n",
      "select_sample subcatalogue time:  5.9496424198150635\n",
      "subfile:  151\n",
      "select_sample subcatalogue time:  5.994293212890625\n",
      "subfile:  152\n",
      "select_sample subcatalogue time:  6.024637937545776\n",
      "subfile:  153\n",
      "select_sample subcatalogue time:  6.063925266265869\n",
      "subfile:  154\n",
      "select_sample subcatalogue time:  6.09546971321106\n",
      "subfile:  155\n",
      "select_sample subcatalogue time:  6.146491050720215\n",
      "subfile:  156\n",
      "select_sample subcatalogue time:  6.1735756397247314\n",
      "subfile:  157\n",
      "select_sample subcatalogue time:  6.219225645065308\n",
      "subfile:  158\n",
      "select_sample subcatalogue time:  6.247469425201416\n",
      "subfile:  159\n",
      "select_sample subcatalogue time:  6.277868032455444\n",
      "subfile:  160\n",
      "select_sample subcatalogue time:  6.321354627609253\n",
      "subfile:  161\n",
      "select_sample subcatalogue time:  6.354302644729614\n",
      "subfile:  162\n",
      "select_sample subcatalogue time:  6.390101432800293\n",
      "subfile:  163\n",
      "select_sample subcatalogue time:  6.4250266551971436\n",
      "subfile:  164\n",
      "select_sample subcatalogue time:  6.462420225143433\n",
      "subfile:  165\n",
      "select_sample subcatalogue time:  6.500575542449951\n",
      "subfile:  166\n",
      "select_sample subcatalogue time:  6.542640686035156\n",
      "subfile:  167\n",
      "select_sample subcatalogue time:  6.5763654708862305\n",
      "subfile:  168\n",
      "select_sample subcatalogue time:  6.6144187450408936\n",
      "subfile:  169\n",
      "select_sample subcatalogue time:  6.647577524185181\n",
      "subfile:  170\n",
      "select_sample subcatalogue time:  6.686738014221191\n",
      "subfile:  171\n",
      "select_sample subcatalogue time:  6.729142665863037\n",
      "subfile:  172\n",
      "select_sample subcatalogue time:  6.759122610092163\n",
      "subfile:  173\n",
      "select_sample subcatalogue time:  6.797452449798584\n",
      "subfile:  174\n",
      "select_sample subcatalogue time:  6.833986282348633\n",
      "subfile:  175\n",
      "select_sample subcatalogue time:  6.868480682373047\n",
      "subfile:  176\n",
      "select_sample subcatalogue time:  6.911544322967529\n",
      "subfile:  177\n",
      "select_sample subcatalogue time:  6.940777778625488\n",
      "subfile:  178\n",
      "select_sample subcatalogue time:  6.978864908218384\n",
      "subfile:  179\n",
      "select_sample subcatalogue time:  7.011072874069214\n",
      "subfile:  180\n",
      "select_sample subcatalogue time:  7.054067373275757\n",
      "subfile:  181\n",
      "select_sample subcatalogue time:  7.081405162811279\n",
      "subfile:  182\n",
      "select_sample subcatalogue time:  7.125887632369995\n",
      "subfile:  183\n",
      "select_sample subcatalogue time:  7.151660680770874\n",
      "subfile:  184\n",
      "select_sample subcatalogue time:  7.212628126144409\n",
      "subfile:  185\n",
      "select_sample subcatalogue time:  7.235854387283325\n",
      "subfile:  186\n",
      "select_sample subcatalogue time:  7.277922868728638\n",
      "subfile:  187\n",
      "select_sample subcatalogue time:  7.304190397262573\n",
      "subfile:  188\n",
      "select_sample subcatalogue time:  7.343433380126953\n",
      "subfile:  189\n",
      "select_sample subcatalogue time:  7.3775200843811035\n",
      "subfile:  190\n",
      "select_sample subcatalogue time:  7.418041706085205\n",
      "subfile:  191\n",
      "select_sample subcatalogue time:  7.453466415405273\n",
      "subfile:  192\n",
      "select_sample subcatalogue time:  7.487894773483276\n",
      "subfile:  193\n",
      "select_sample subcatalogue time:  7.527709722518921\n",
      "subfile:  194\n",
      "select_sample subcatalogue time:  7.566932201385498\n",
      "subfile:  195\n",
      "select_sample subcatalogue time:  7.598510503768921\n",
      "subfile:  196\n",
      "select_sample subcatalogue time:  7.6399266719818115\n",
      "subfile:  197\n",
      "select_sample subcatalogue time:  7.6604390144348145\n",
      "subfile:  198\n",
      "select_sample subcatalogue time:  7.706047296524048\n",
      "subfile:  199\n",
      "select_sample subcatalogue time:  7.740370035171509\n",
      "subfile:  200\n",
      "select_sample subcatalogue time:  7.788177490234375\n",
      "subfile:  201\n",
      "select_sample subcatalogue time:  7.819976329803467\n",
      "subfile:  202\n",
      "select_sample subcatalogue time:  7.860043048858643\n",
      "subfile:  203\n",
      "select_sample subcatalogue time:  7.893399953842163\n",
      "subfile:  204\n",
      "select_sample subcatalogue time:  7.923756122589111\n",
      "subfile:  205\n",
      "select_sample subcatalogue time:  7.965369462966919\n",
      "subfile:  206\n",
      "select_sample subcatalogue time:  8.006974220275879\n",
      "subfile:  207\n",
      "select_sample subcatalogue time:  8.038447856903076\n",
      "subfile:  208\n",
      "select_sample subcatalogue time:  8.075601577758789\n",
      "subfile:  209\n",
      "select_sample subcatalogue time:  8.109168291091919\n",
      "subfile:  210\n",
      "select_sample subcatalogue time:  8.153258085250854\n",
      "subfile:  211\n",
      "select_sample subcatalogue time:  8.174967765808105\n",
      "subfile:  212\n",
      "select_sample subcatalogue time:  8.222638130187988\n",
      "subfile:  213\n",
      "select_sample subcatalogue time:  8.256407499313354\n",
      "subfile:  214\n",
      "select_sample subcatalogue time:  8.303458452224731\n",
      "subfile:  215\n",
      "select_sample subcatalogue time:  8.329735279083252\n",
      "subfile:  216\n",
      "select_sample subcatalogue time:  8.36440658569336\n",
      "subfile:  217\n",
      "select_sample subcatalogue time:  8.410160779953003\n",
      "subfile:  218\n",
      "select_sample subcatalogue time:  8.44046950340271\n",
      "subfile:  219\n",
      "select_sample subcatalogue time:  8.473142147064209\n",
      "subfile:  220\n",
      "select_sample subcatalogue time:  8.519151449203491\n",
      "subfile:  221\n",
      "select_sample subcatalogue time:  8.544537782669067\n",
      "subfile:  222\n",
      "select_sample subcatalogue time:  8.591577768325806\n",
      "subfile:  223\n",
      "select_sample subcatalogue time:  8.622269868850708\n",
      "subfile:  224\n",
      "select_sample subcatalogue time:  8.659962177276611\n",
      "subfile:  225\n",
      "select_sample subcatalogue time:  8.695080518722534\n",
      "subfile:  226\n",
      "select_sample subcatalogue time:  8.734734058380127\n",
      "subfile:  227\n",
      "select_sample subcatalogue time:  8.762712001800537\n",
      "subfile:  228\n",
      "select_sample subcatalogue time:  8.80253267288208\n",
      "subfile:  229\n",
      "select_sample subcatalogue time:  8.84226131439209\n",
      "subfile:  230\n",
      "select_sample subcatalogue time:  8.886806726455688\n",
      "subfile:  231\n",
      "select_sample subcatalogue time:  8.90875506401062\n",
      "subfile:  232\n",
      "select_sample subcatalogue time:  8.944843292236328\n",
      "subfile:  233\n",
      "select_sample subcatalogue time:  8.986717700958252\n",
      "subfile:  234\n",
      "select_sample subcatalogue time:  9.038283824920654\n",
      "subfile:  235\n",
      "select_sample subcatalogue time:  9.056883811950684\n",
      "subfile:  236\n",
      "select_sample subcatalogue time:  9.096726417541504\n",
      "subfile:  237\n",
      "select_sample subcatalogue time:  9.130318641662598\n",
      "subfile:  238\n",
      "select_sample subcatalogue time:  9.167367458343506\n",
      "subfile:  239\n",
      "select_sample subcatalogue time:  9.207331657409668\n",
      "subfile:  240\n",
      "select_sample subcatalogue time:  9.252162456512451\n",
      "subfile:  241\n",
      "select_sample subcatalogue time:  9.279558897018433\n",
      "subfile:  242\n",
      "select_sample subcatalogue time:  9.314393758773804\n",
      "subfile:  243\n",
      "select_sample subcatalogue time:  9.348255395889282\n",
      "subfile:  244\n",
      "select_sample subcatalogue time:  9.393003463745117\n",
      "subfile:  245\n",
      "select_sample subcatalogue time:  9.418540716171265\n",
      "subfile:  246\n",
      "select_sample subcatalogue time:  9.459224462509155\n",
      "subfile:  247\n",
      "select_sample subcatalogue time:  9.505498170852661\n",
      "subfile:  248\n",
      "select_sample subcatalogue time:  9.54590392112732\n",
      "subfile:  249\n",
      "select_sample subcatalogue time:  9.56687593460083\n",
      "subfile:  250\n",
      "select_sample subcatalogue time:  9.610942602157593\n",
      "subfile:  251\n",
      "select_sample subcatalogue time:  9.635379076004028\n",
      "subfile:  252\n",
      "select_sample subcatalogue time:  9.680207967758179\n",
      "subfile:  253\n",
      "select_sample subcatalogue time:  9.71408748626709\n",
      "subfile:  254\n",
      "select_sample subcatalogue time:  9.774112939834595\n",
      "subfile:  255\n",
      "select_sample subcatalogue time:  9.78943133354187\n",
      "subfile:  256\n",
      "select_sample subcatalogue time:  9.828409671783447\n",
      "subfile:  257\n",
      "select_sample subcatalogue time:  9.86758041381836\n",
      "subfile:  258\n",
      "select_sample subcatalogue time:  9.913543701171875\n",
      "subfile:  259\n",
      "select_sample subcatalogue time:  9.938554286956787\n",
      "subfile:  260\n",
      "select_sample subcatalogue time:  9.976763486862183\n",
      "subfile:  261\n",
      "select_sample subcatalogue time:  9.997021913528442\n",
      "subfile:  262\n",
      "select_sample subcatalogue time:  10.045685529708862\n",
      "subfile:  263\n",
      "select_sample subcatalogue time:  10.083924293518066\n",
      "subfile:  264\n",
      "select_sample subcatalogue time:  10.129563570022583\n",
      "subfile:  265\n",
      "select_sample subcatalogue time:  10.153462886810303\n",
      "subfile:  266\n",
      "select_sample subcatalogue time:  10.192347049713135\n",
      "subfile:  267\n",
      "select_sample subcatalogue time:  10.22416090965271\n",
      "subfile:  268\n",
      "select_sample subcatalogue time:  10.277443885803223\n",
      "subfile:  269\n",
      "select_sample subcatalogue time:  10.314920902252197\n",
      "subfile:  270\n",
      "select_sample subcatalogue time:  10.345110893249512\n",
      "subfile:  271\n",
      "select_sample subcatalogue time:  10.370349645614624\n",
      "subfile:  272\n",
      "select_sample subcatalogue time:  10.416579008102417\n",
      "subfile:  273\n",
      "select_sample subcatalogue time:  10.451332330703735\n",
      "subfile:  274\n",
      "select_sample subcatalogue time:  10.497388362884521\n",
      "subfile:  275\n",
      "select_sample subcatalogue time:  10.513033390045166\n",
      "subfile:  276\n",
      "select_sample subcatalogue time:  10.55717158317566\n",
      "subfile:  277\n",
      "select_sample subcatalogue time:  10.592117547988892\n",
      "subfile:  278\n",
      "select_sample subcatalogue time:  10.647334098815918\n",
      "subfile:  279\n",
      "select_sample subcatalogue time:  10.665273189544678\n",
      "subfile:  280\n",
      "select_sample subcatalogue time:  10.696540355682373\n",
      "subfile:  281\n",
      "select_sample subcatalogue time:  10.753768682479858\n",
      "subfile:  282\n",
      "select_sample subcatalogue time:  10.779610395431519\n",
      "subfile:  283\n",
      "select_sample subcatalogue time:  10.811259031295776\n",
      "subfile:  284\n",
      "select_sample subcatalogue time:  10.857993125915527\n",
      "subfile:  285\n",
      "select_sample subcatalogue time:  10.877582550048828\n",
      "subfile:  286\n",
      "select_sample subcatalogue time:  10.932967901229858\n",
      "subfile:  287\n",
      "select_sample subcatalogue time:  10.96225094795227\n",
      "subfile:  288\n",
      "select_sample subcatalogue time:  11.00502610206604\n",
      "subfile:  289\n",
      "select_sample subcatalogue time:  11.024614810943604\n",
      "subfile:  290\n",
      "select_sample subcatalogue time:  11.069073915481567\n",
      "subfile:  291\n",
      "select_sample subcatalogue time:  11.102329730987549\n",
      "subfile:  292\n",
      "select_sample subcatalogue time:  11.154993057250977\n",
      "subfile:  293\n",
      "select_sample subcatalogue time:  11.175745010375977\n",
      "subfile:  294\n",
      "select_sample subcatalogue time:  11.223219156265259\n",
      "subfile:  295\n",
      "select_sample subcatalogue time:  11.244060277938843\n",
      "subfile:  296\n",
      "select_sample subcatalogue time:  11.292784452438354\n",
      "subfile:  297\n",
      "select_sample subcatalogue time:  11.341525554656982\n",
      "subfile:  298\n",
      "select_sample subcatalogue time:  11.373876333236694\n",
      "subfile:  299\n",
      "select_sample subcatalogue time:  11.396687984466553\n",
      "subfile:  300\n",
      "select_sample subcatalogue time:  11.431952953338623\n",
      "subfile:  301\n",
      "select_sample subcatalogue time:  11.478364706039429\n",
      "subfile:  302\n",
      "select_sample subcatalogue time:  11.514665365219116\n",
      "subfile:  303\n",
      "select_sample subcatalogue time:  11.536953210830688\n",
      "subfile:  304\n",
      "select_sample subcatalogue time:  11.579668045043945\n",
      "subfile:  305\n",
      "select_sample subcatalogue time:  11.610745191574097\n",
      "subfile:  306\n",
      "select_sample subcatalogue time:  11.650487661361694\n",
      "subfile:  307\n",
      "select_sample subcatalogue time:  11.68386721611023\n",
      "subfile:  308\n",
      "select_sample subcatalogue time:  11.728502988815308\n",
      "subfile:  309\n",
      "select_sample subcatalogue time:  11.75084638595581\n",
      "subfile:  310\n",
      "select_sample subcatalogue time:  11.794610977172852\n",
      "subfile:  311\n",
      "select_sample subcatalogue time:  11.834158420562744\n",
      "subfile:  312\n",
      "select_sample subcatalogue time:  11.880038022994995\n",
      "subfile:  313\n",
      "select_sample subcatalogue time:  11.90876841545105\n",
      "subfile:  314\n",
      "select_sample subcatalogue time:  11.9479341506958\n",
      "subfile:  315\n",
      "select_sample subcatalogue time:  11.979010343551636\n",
      "subfile:  316\n",
      "select_sample subcatalogue time:  12.018919706344604\n",
      "subfile:  317\n",
      "select_sample subcatalogue time:  12.05138349533081\n",
      "subfile:  318\n",
      "select_sample subcatalogue time:  12.099369287490845\n",
      "subfile:  319\n",
      "select_sample subcatalogue time:  12.116420030593872\n",
      "subfile:  320\n",
      "select_sample subcatalogue time:  12.168488264083862\n",
      "subfile:  321\n",
      "select_sample subcatalogue time:  12.19826626777649\n",
      "subfile:  322\n",
      "select_sample subcatalogue time:  12.24780011177063\n",
      "subfile:  323\n",
      "select_sample subcatalogue time:  12.269800186157227\n",
      "subfile:  324\n",
      "select_sample subcatalogue time:  12.312525033950806\n",
      "subfile:  325\n",
      "select_sample subcatalogue time:  12.345303773880005\n",
      "subfile:  326\n",
      "select_sample subcatalogue time:  12.38831353187561\n",
      "subfile:  327\n",
      "select_sample subcatalogue time:  12.416139841079712\n",
      "subfile:  328\n",
      "select_sample subcatalogue time:  12.464777708053589\n",
      "subfile:  329\n",
      "select_sample subcatalogue time:  12.481739282608032\n",
      "subfile:  330\n",
      "select_sample subcatalogue time:  12.531667947769165\n",
      "subfile:  331\n",
      "select_sample subcatalogue time:  12.559616565704346\n",
      "subfile:  332\n",
      "select_sample subcatalogue time:  12.611612796783447\n",
      "subfile:  333\n",
      "select_sample subcatalogue time:  12.649093389511108\n",
      "subfile:  334\n",
      "select_sample subcatalogue time:  12.679437637329102\n",
      "subfile:  335\n",
      "select_sample subcatalogue time:  12.700669288635254\n",
      "subfile:  336\n",
      "select_sample subcatalogue time:  12.75450611114502\n",
      "subfile:  337\n",
      "select_sample subcatalogue time:  12.783679485321045\n",
      "subfile:  338\n",
      "select_sample subcatalogue time:  12.832334041595459\n",
      "subfile:  339\n",
      "select_sample subcatalogue time:  12.846095085144043\n",
      "subfile:  340\n",
      "select_sample subcatalogue time:  12.888607263565063\n",
      "subfile:  341\n",
      "select_sample subcatalogue time:  12.930461406707764\n",
      "subfile:  342\n",
      "select_sample subcatalogue time:  12.973761558532715\n",
      "subfile:  343\n",
      "select_sample subcatalogue time:  12.997063636779785\n",
      "subfile:  344\n",
      "select_sample subcatalogue time:  13.045091152191162\n",
      "subfile:  345\n",
      "select_sample subcatalogue time:  13.073455572128296\n",
      "subfile:  346\n",
      "select_sample subcatalogue time:  13.121838808059692\n",
      "subfile:  347\n",
      "select_sample subcatalogue time:  13.146301031112671\n",
      "subfile:  348\n",
      "select_sample subcatalogue time:  13.191385746002197\n",
      "subfile:  349\n",
      "select_sample subcatalogue time:  13.210529088973999\n",
      "subfile:  350\n",
      "select_sample subcatalogue time:  13.263245344161987\n",
      "subfile:  351\n",
      "select_sample subcatalogue time:  13.292623281478882\n",
      "subfile:  352\n",
      "select_sample subcatalogue time:  13.353915214538574\n",
      "subfile:  353\n",
      "select_sample subcatalogue time:  13.357041597366333\n",
      "subfile:  354\n",
      "select_sample subcatalogue time:  13.411645650863647\n",
      "subfile:  355\n",
      "select_sample subcatalogue time:  13.434548377990723\n",
      "subfile:  356\n",
      "select_sample subcatalogue time:  13.490944385528564\n",
      "subfile:  357\n",
      "select_sample subcatalogue time:  13.504848003387451\n",
      "subfile:  358\n",
      "select_sample subcatalogue time:  13.556272506713867\n",
      "subfile:  359\n",
      "select_sample subcatalogue time:  13.571063995361328\n",
      "subfile:  360\n",
      "select_sample subcatalogue time:  13.61630654335022\n",
      "subfile:  361\n",
      "select_sample subcatalogue time:  13.663927793502808\n",
      "subfile:  362\n",
      "select_sample subcatalogue time:  13.703722715377808\n",
      "subfile:  363\n",
      "select_sample subcatalogue time:  13.749657392501831\n",
      "subfile:  364\n",
      "select_sample subcatalogue time:  13.772361993789673\n",
      "subfile:  365\n",
      "select_sample subcatalogue time:  13.805993795394897\n",
      "subfile:  366\n",
      "select_sample subcatalogue time:  13.859659671783447\n",
      "subfile:  367\n",
      "select_sample subcatalogue time:  13.873272180557251\n",
      "subfile:  368\n",
      "select_sample subcatalogue time:  13.92007827758789\n",
      "subfile:  369\n",
      "select_sample subcatalogue time:  13.957497596740723\n",
      "subfile:  370\n",
      "select_sample subcatalogue time:  14.007044792175293\n",
      "subfile:  371\n",
      "select_sample subcatalogue time:  14.052095890045166\n",
      "subfile:  372\n",
      "select_sample subcatalogue time:  14.06883716583252\n",
      "subfile:  373\n",
      "select_sample subcatalogue time:  14.094937086105347\n",
      "subfile:  374\n",
      "select_sample subcatalogue time:  14.134888410568237\n",
      "subfile:  375\n",
      "select_sample subcatalogue time:  14.162856578826904\n",
      "subfile:  376\n",
      "select_sample subcatalogue time:  14.21756386756897\n",
      "subfile:  377\n",
      "select_sample subcatalogue time:  14.239871501922607\n",
      "subfile:  378\n",
      "select_sample subcatalogue time:  14.284053802490234\n",
      "subfile:  379\n",
      "select_sample subcatalogue time:  14.346346616744995\n",
      "subfile:  380\n",
      "select_sample subcatalogue time:  14.353851556777954\n",
      "subfile:  381\n",
      "select_sample subcatalogue time:  14.387859106063843\n",
      "subfile:  382\n",
      "select_sample subcatalogue time:  14.454707860946655\n",
      "subfile:  383\n",
      "select_sample subcatalogue time:  14.453312158584595\n",
      "subfile:  384\n",
      "select_sample subcatalogue time:  14.50793194770813\n",
      "subfile:  385\n",
      "select_sample subcatalogue time:  14.530078649520874\n",
      "subfile:  386\n",
      "select_sample subcatalogue time:  14.583629846572876\n",
      "subfile:  387\n",
      "select_sample subcatalogue time:  14.610493659973145\n",
      "subfile:  388\n",
      "select_sample subcatalogue time:  14.645666360855103\n",
      "subfile:  389\n",
      "select_sample subcatalogue time:  14.688118696212769\n",
      "subfile:  390\n",
      "select_sample subcatalogue time:  14.72705626487732\n",
      "subfile:  391\n",
      "select_sample subcatalogue time:  14.752119064331055\n",
      "subfile:  392\n",
      "select_sample subcatalogue time:  14.799606561660767\n",
      "subfile:  393\n",
      "select_sample subcatalogue time:  14.815365076065063\n",
      "subfile:  394\n",
      "select_sample subcatalogue time:  14.872415542602539\n",
      "subfile:  395\n",
      "select_sample subcatalogue time:  14.898933172225952\n",
      "subfile:  396\n",
      "select_sample subcatalogue time:  14.950467586517334\n",
      "subfile:  397\n",
      "select_sample subcatalogue time:  14.96653699874878\n",
      "subfile:  398\n",
      "select_sample subcatalogue time:  15.008430480957031\n",
      "subfile:  399\n",
      "select_sample subcatalogue time:  15.05305528640747\n",
      "subfile:  400\n",
      "select_sample subcatalogue time:  15.093348979949951\n",
      "subfile:  401\n",
      "select_sample subcatalogue time:  15.11098337173462\n",
      "subfile:  402\n",
      "select_sample subcatalogue time:  15.163144826889038\n",
      "subfile:  403\n",
      "select_sample subcatalogue time:  15.182816505432129\n",
      "subfile:  404\n",
      "select_sample subcatalogue time:  15.235465049743652\n",
      "subfile:  405\n",
      "select_sample subcatalogue time:  15.26200008392334\n",
      "subfile:  406\n",
      "select_sample subcatalogue time:  15.31017804145813\n",
      "subfile:  407\n",
      "select_sample subcatalogue time:  15.327816247940063\n",
      "subfile:  408\n",
      "select_sample subcatalogue time:  15.37549090385437\n",
      "subfile:  409\n",
      "select_sample subcatalogue time:  15.414304256439209\n",
      "subfile:  410\n",
      "select_sample subcatalogue time:  15.459082126617432\n",
      "subfile:  411\n",
      "select_sample subcatalogue time:  15.481302738189697\n",
      "subfile:  412\n",
      "select_sample subcatalogue time:  15.528144598007202\n",
      "subfile:  413\n",
      "select_sample subcatalogue time:  15.543015956878662\n",
      "subfile:  414\n",
      "select_sample subcatalogue time:  15.596591472625732\n",
      "subfile:  415\n",
      "select_sample subcatalogue time:  15.63684368133545\n",
      "subfile:  416\n",
      "select_sample subcatalogue time:  15.677070379257202\n",
      "subfile:  417\n",
      "select_sample subcatalogue time:  15.694786787033081\n",
      "subfile:  418\n",
      "select_sample subcatalogue time:  15.74643588066101\n",
      "subfile:  419\n",
      "select_sample subcatalogue time:  15.77892279624939\n",
      "subfile:  420\n",
      "select_sample subcatalogue time:  15.829972982406616\n",
      "subfile:  421\n",
      "select_sample subcatalogue time:  15.840227603912354\n",
      "subfile:  422\n",
      "select_sample subcatalogue time:  15.884186029434204\n",
      "subfile:  423\n",
      "select_sample subcatalogue time:  15.931246519088745\n",
      "subfile:  424\n",
      "select_sample subcatalogue time:  15.956822872161865\n",
      "subfile:  425\n",
      "select_sample subcatalogue time:  15.99306058883667\n",
      "subfile:  426\n",
      "select_sample subcatalogue time:  16.043713331222534\n",
      "subfile:  427\n",
      "select_sample subcatalogue time:  16.057714700698853\n",
      "subfile:  428\n",
      "select_sample subcatalogue time:  16.10598587989807\n",
      "subfile:  429\n",
      "select_sample subcatalogue time:  16.14333748817444\n",
      "subfile:  430\n",
      "select_sample subcatalogue time:  16.1973557472229\n",
      "subfile:  431\n",
      "select_sample subcatalogue time:  16.229355096817017\n",
      "subfile:  432\n",
      "select_sample subcatalogue time:  16.25382137298584\n",
      "subfile:  433\n",
      "select_sample subcatalogue time:  16.28439712524414\n",
      "subfile:  434\n",
      "select_sample subcatalogue time:  16.334709882736206\n",
      "subfile:  435\n",
      "select_sample subcatalogue time:  16.38965392112732\n",
      "subfile:  436\n",
      "select_sample subcatalogue time:  16.409658193588257\n",
      "subfile:  437\n",
      "select_sample subcatalogue time:  16.422918796539307\n",
      "subfile:  438\n",
      "select_sample subcatalogue time:  16.47534155845642\n",
      "subfile:  439\n",
      "select_sample subcatalogue time:  16.502795696258545\n",
      "subfile:  440\n",
      "select_sample subcatalogue time:  16.561153411865234\n",
      "subfile:  441\n",
      "select_sample subcatalogue time:  16.571712493896484\n",
      "subfile:  442\n",
      "select_sample subcatalogue time:  16.62587261199951\n",
      "subfile:  443\n",
      "select_sample subcatalogue time:  16.657106399536133\n",
      "subfile:  444\n",
      "select_sample subcatalogue time:  16.70705556869507\n",
      "subfile:  445\n",
      "select_sample subcatalogue time:  16.722671508789062\n",
      "subfile:  446\n",
      "select_sample subcatalogue time:  16.80105972290039\n",
      "subfile:  447\n",
      "select_sample subcatalogue time:  16.7903790473938\n",
      "subfile:  448\n",
      "select_sample subcatalogue time:  16.83800768852234\n",
      "subfile:  449\n",
      "select_sample subcatalogue time:  16.876720428466797\n",
      "subfile:  450\n",
      "select_sample subcatalogue time:  16.92305636405945\n",
      "subfile:  451\n",
      "select_sample subcatalogue time:  16.937528133392334\n",
      "subfile:  452\n",
      "select_sample subcatalogue time:  16.98596215248108\n",
      "subfile:  453\n",
      "select_sample subcatalogue time:  17.027478218078613\n",
      "subfile:  454\n",
      "select_sample subcatalogue time:  17.07896637916565\n",
      "subfile:  455\n",
      "select_sample subcatalogue time:  17.084825038909912\n",
      "subfile:  456\n",
      "select_sample subcatalogue time:  17.132167100906372\n",
      "subfile:  457\n",
      "select_sample subcatalogue time:  17.158525466918945\n",
      "subfile:  458\n",
      "select_sample subcatalogue time:  17.204535007476807\n",
      "subfile:  459\n",
      "select_sample subcatalogue time:  17.2409884929657\n",
      "subfile:  460\n",
      "select_sample subcatalogue time:  17.295836925506592\n",
      "subfile:  461\n",
      "select_sample subcatalogue time:  17.298930883407593\n",
      "subfile:  462\n",
      "select_sample subcatalogue time:  17.356401920318604\n",
      "subfile:  463\n",
      "select_sample subcatalogue time:  17.385774612426758\n",
      "subfile:  464\n",
      "select_sample subcatalogue time:  17.439188957214355\n",
      "subfile:  465\n",
      "select_sample subcatalogue time:  17.452277660369873\n",
      "subfile:  466\n",
      "select_sample subcatalogue time:  17.50153660774231\n",
      "subfile:  467\n",
      "select_sample subcatalogue time:  17.52189540863037\n",
      "subfile:  468\n",
      "select_sample subcatalogue time:  17.566102981567383\n",
      "subfile:  469\n",
      "select_sample subcatalogue time:  17.603294849395752\n",
      "subfile:  470\n",
      "select_sample subcatalogue time:  17.646084785461426\n",
      "subfile:  471\n",
      "select_sample subcatalogue time:  17.669801950454712\n",
      "subfile:  472\n",
      "select_sample subcatalogue time:  17.715112447738647\n",
      "subfile:  473\n",
      "select_sample subcatalogue time:  17.756797552108765\n",
      "subfile:  474\n",
      "select_sample subcatalogue time:  17.809646129608154\n",
      "subfile:  475\n",
      "select_sample subcatalogue time:  17.815415143966675\n",
      "subfile:  476\n",
      "select_sample subcatalogue time:  17.864001035690308\n",
      "subfile:  477\n",
      "select_sample subcatalogue time:  17.89539861679077\n",
      "subfile:  478\n",
      "select_sample subcatalogue time:  17.94989323616028\n",
      "subfile:  479\n",
      "select_sample subcatalogue time:  17.969024896621704\n",
      "subfile:  480\n",
      "select_sample subcatalogue time:  18.009922742843628\n",
      "subfile:  481\n",
      "select_sample subcatalogue time:  18.01941466331482\n",
      "subfile:  482\n",
      "select_sample subcatalogue time:  18.076812028884888\n",
      "subfile:  483\n",
      "select_sample subcatalogue time:  18.116440057754517\n",
      "subfile:  484\n",
      "select_sample subcatalogue time:  18.158242225646973\n",
      "subfile:  485\n",
      "select_sample subcatalogue time:  18.17604684829712\n",
      "subfile:  486\n",
      "select_sample subcatalogue time:  18.225402355194092\n",
      "subfile:  487\n",
      "select_sample subcatalogue time:  18.25549578666687\n",
      "subfile:  488\n",
      "select_sample subcatalogue time:  18.305376052856445\n",
      "subfile:  489\n",
      "select_sample subcatalogue time:  18.32980179786682\n",
      "subfile:  490\n",
      "select_sample subcatalogue time:  18.376063108444214\n",
      "subfile:  491\n",
      "select_sample subcatalogue time:  18.393842697143555\n",
      "subfile:  492\n",
      "select_sample subcatalogue time:  18.443932056427002\n",
      "subfile:  493\n",
      "select_sample subcatalogue time:  18.47207283973694\n",
      "subfile:  494\n",
      "select_sample subcatalogue time:  18.511833667755127\n",
      "subfile:  495\n",
      "select_sample subcatalogue time:  18.540319442749023\n",
      "subfile:  496\n",
      "select_sample subcatalogue time:  18.593347787857056\n",
      "subfile:  497\n",
      "select_sample subcatalogue time:  19.811107873916626\n",
      "subfile:  498\n",
      "select_sample subcatalogue time:  18.741581678390503\n",
      "subfile:  499\n",
      "select_sample subcatalogue time:  18.72859787940979\n",
      "subfile:  500\n",
      "select_sample subcatalogue time:  18.78498387336731\n",
      "subfile:  501\n",
      "select_sample subcatalogue time:  18.796226739883423\n",
      "select_sample total time:  4853.740289211273\n"
     ]
    }
   ],
   "source": [
    "if use_subfiles == False:\n",
    "    ind_cat = select_sample(masterlist, helpbands, moc_list, mcuts)\n",
    "\n",
    "else:\n",
    "    ind_cat = select_sample_subfiles(file_list, helpbands, moc_list, mcuts)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "ind_cat.write('./data/ind_cat_sf_{}.fits'.format(SUFFIX),overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## II Compare with Ldust catalogue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "ldustfile = DMU_DIR + 'dmu28/dmu28_Herschel-Stripe-82/data/Ldust/old/herschel_stripe_82_Ldust_prediction_all.fits'\n",
    "#ldust_cat = Table.read(ldustfile, memmap=True)\n",
    "#ldust_cat['id'].name = 'help_id'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# indexs catalogues based on flags\n",
    "#ind_catalogue_ldustcriteria = (ind_catalogue['ndet_total']==3) | (ind_catalogue['ndet_total']==7)\n",
    "# ind_catalogue_iraccriteria = ind_catalogue['ndet_total']>=5\n",
    "\n",
    "\n",
    "# Compare sources after selection criteria with ldust catalogue \n",
    "# Plotting Venn diagrams \n",
    "#########################################\n",
    "#ind_catalogues = np.array([\n",
    "#    ind_catalogue_ldustcriteria,\n",
    "#     ind_catalogue_iraccriteria\n",
    "#    ])\n",
    "\n",
    "#ind_names =  np.array([\n",
    "#    '_ldustcriteria_{}'.format(SUFFIX),\n",
    "#     cat_name+'_iraccriteria'\n",
    "#])\n",
    "\n",
    "# Venn Diagrams\n",
    "#compare_cats_venndiagram(ldust_cat['help_id'], ind_catalogue['help_id'], ind_catalogues, ind_names, 'ldust', 'sf')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## III Selected catalogue"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1 Define the final sample\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hs82_cat_selected = masterlist[ind_catalogue_ldustcriteria]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# len(hs82_cat_selected)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hs82_cat_selected.write('./data/hs82_selected_sources_sf_{}.fits'.format(SUFFIX),overwrite=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (herschelhelp_internal)",
   "language": "python",
   "name": "helpint"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}