{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Write speczs to the VO\n",
    "\n",
    "Perform some minor changes to the tables so that they can be ingested to the VO."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This notebook was run with herschelhelp_internal version: \n",
      "017bb1e (Mon Jun 18 14:58:59 2018 +0100)\n",
      "This notebook was executed on: \n",
      "2018-07-20 12:52:12.810625\n"
     ]
    }
   ],
   "source": [
    "from  herschelhelp_internal  import git_version\n",
    "print(\"This notebook was run with herschelhelp_internal version: \\n{}\".format(git_version()))\n",
    "import datetime\n",
    "print(\"This notebook was executed on: \\n{}\".format(datetime.datetime.now()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from astropy.table import Table, Column\n",
    "from herschelhelp_internal.utils import coords_to_hpidx,  gen_help_id\n",
    "\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vo_folder = '/mnt/hedam/data_vo/specz/'\n",
    "#vo_folder = '/Users/rs548/GitHub/gavo_inputs/specz/data/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def convert_specz(original, field, vo_folder):\n",
    "    new = Table.read(original)\n",
    "    new['RA'].name = 'ra'\n",
    "    new['DEC'].name = 'dec'\n",
    "    new['Z_SPEC'].name = 'z_spec'\n",
    "    new['Z_SOURCE'].name = 'z_source'\n",
    "    new['Z_QUAL'].name = 'z_qual'\n",
    "    new['REL'].name = 'z_rel'\n",
    "    new['OBJID'].name = 'objid'\n",
    "    new['AGN'].name = 'agn_flag'\n",
    "    new.add_column(Column(np.char.replace(np.array(gen_help_id(nep['ra'], nep['dec']), dtype=np.string_), b'HELP', b'HELP_SPECZ'),\n",
    "                                   name=\"specz_id\"))\n",
    "    new.add_column(Column(np.full(len(new), field, dtype='<U18'),\n",
    "                                   name=\"field\"))\n",
    "    nep.write(vo_folder + original.split('/')[-1].replace('.csv', '_vo.csv'), overwrite=True)\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'AKARI-NEP-specz-v2_vo.csv'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'./dmu23_AKARI-NEP/data/AKARI-NEP-specz-v2.csv'.split('/')[-1].replace('.csv', '_vo.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## AKARI-NEP\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "nep = Table.read('./dmu23_AKARI-NEP/data/AKARI-NEP-specz-v2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "nep['RA'].name = 'ra'\n",
    "nep['DEC'].name = 'dec'\n",
    "nep['Z_SPEC'].name = 'z_spec'\n",
    "nep['Z_SOURCE'].name = 'z_source'\n",
    "nep['Z_QUAL'].name = 'z_qual'\n",
    "nep['REL'].name = 'z_rel'\n",
    "nep['OBJID'].name = 'objid'\n",
    "nep['AGN'].name = 'agn_flag'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "nep.add_column(Column(np.char.replace(np.array(gen_help_id(nep['ra'], nep['dec']), dtype=np.string_), b'HELP', b'HELP_SPECZ'),\n",
    "                                   name=\"specz_id\"))\n",
    "nep.add_column(Column(np.full(len(nep), \"AKARI-NEP\", dtype='<U18'),\n",
    "                                   name=\"field\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "nep.write(vo_folder + 'AKARI-NEP-specz-v2_vo.csv', overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AKARI-SEP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sep  = Table.read('./dmu23_AKARI-SEP/data/Akari-SEP-specz-v2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "sep['RA'].name = 'ra'\n",
    "sep['DEC'].name = 'dec'\n",
    "sep['Z_SPEC'].name = 'z_spec'\n",
    "sep['Z_SOURCE'].name = 'z_source'\n",
    "sep['Z_QUAL'].name = 'z_qual'\n",
    "sep['REL'].name = 'z_rel'\n",
    "sep['OBJID'].name = 'objid'\n",
    "sep['AGN'].name = 'agn_flag'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sep.add_column(Column(np.char.replace(np.array(gen_help_id(sep['ra'], sep['dec']), dtype=np.string_), b'HELP', b'HELP_SPECZ'),\n",
    "                                   name=\"specz_id\"))\n",
    "sep.add_column(Column(np.full(len(sep), \"AKARI-SEP\", dtype='<U18'),\n",
    "                                   name=\"field\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "sep.write(vo_folder + 'AKARI-SEP-specz-v2_vo.csv', overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## HDFN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "hdfn  = Table.read('./dmu23_HDF-N/data/HDF-N-specz-v2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "hdfn['RA'].name = 'ra'\n",
    "hdfn['DEC'].name = 'dec'\n",
    "hdfn['Z_SPEC'].name = 'z_spec'\n",
    "hdfn['Z_SOURCE'].name = 'z_source'\n",
    "hdfn['Z_QUAL'].name = 'z_qual'\n",
    "hdfn['REL'].name = 'z_rel'\n",
    "hdfn['OBJID'].name = 'objid'\n",
    "hdfn['AGN'].name = 'agn_flag'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "hdfn.add_column(Column(np.char.replace(np.array(gen_help_id(hdfn['ra'], hdfn['dec']), dtype=np.string_), b'HELP', b'HELP_SPECZ'),\n",
    "                                   name=\"specz_id\"))\n",
    "hdfn.add_column(Column(np.full(len(hdfn), \"HDF-N\", dtype='<U18'),\n",
    "                                   name=\"field\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "hdfn.write(vo_folder + 'HDF-N-specz-v2_vo.csv', overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SA13"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sa13  = Table.read('./dmu23_SA13/data/SA13-specz-v2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "sa13['RA'].name = 'ra'\n",
    "sa13['DEC'].name = 'dec'\n",
    "sa13['Z_SPEC'].name = 'z_spec'\n",
    "sa13['Z_SOURCE'].name = 'z_source'\n",
    "sa13['Z_QUAL'].name = 'z_qual'\n",
    "sa13['REL'].name = 'z_rel'\n",
    "sa13['OBJID'].name = 'objid'\n",
    "sa13['AGN'].name = 'agn_flag'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sa13.add_column(Column(np.char.replace(np.array(gen_help_id(sa13['ra'], sa13['dec']), dtype=np.string_), b'HELP', b'HELP_SPECZ'),\n",
    "                                   name=\"specz_id\"))\n",
    "sa13.add_column(Column(np.full(len(sa13), \"HDF-N\", dtype='<U18'),\n",
    "                                   name=\"field\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sa13.write(vo_folder + 'SA13-specz-v2_vo.csv', overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SPIRE-NEP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nep  = Table.read('./dmu23_SPIRE-NEP/data/SPIRE-NEP-specz-v2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nep['RA'].name = 'ra'\n",
    "nep['DEC'].name = 'dec'\n",
    "nep['Z_SPEC'].name = 'z_spec'\n",
    "nep['Z_SOURCE'].name = 'z_source'\n",
    "nep['Z_QUAL'].name = 'z_qual'\n",
    "nep['REL'].name = 'z_rel'\n",
    "nep['OBJID'].name = 'objid'\n",
    "nep['AGN'].name = 'agn_flag'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nep.add_column(Column(np.char.replace(np.array(gen_help_id(nep['ra'], nep['dec']), dtype=np.string_), b'HELP', b'HELP_SPECZ'),\n",
    "                                   name=\"specz_id\"))\n",
    "nep.add_column(Column(np.full(len(nep), \"SPIRE-NEP\", dtype='<U18'),\n",
    "                                   name=\"field\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nep.write(vo_folder + 'SPIRE-NEP-specz-v2_vo.csv', overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XMM-13hr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "xmm13  = Table.read('./dmu23_XMM-13hr/data/XMM-13hr-specz-v2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xmm13['RA'].name = 'ra'\n",
    "xmm13['DEC'].name = 'dec'\n",
    "xmm13['Z_SPEC'].name = 'z_spec'\n",
    "xmm13['Z_SOURCE'].name = 'z_source'\n",
    "xmm13['Z_QUAL'].name = 'z_qual'\n",
    "xmm13['REL'].name = 'z_rel'\n",
    "xmm13['OBJID'].name = 'objid'\n",
    "xmm13['AGN'].name = 'agn_flag'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "xmm13.add_column(Column(np.char.replace(np.array(gen_help_id(xmm13['ra'], xmm13['dec']), dtype=np.string_), b'HELP', b'HELP_SPECZ'),\n",
    "                                   name=\"specz_id\"))\n",
    "xmm13.add_column(Column(np.full(len(xmm13), \"XMM-13hr\", dtype='<U18'),\n",
    "                                   name=\"field\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xmm13.write(vo_folder + 'XMM-13hr-specz-v2_vo.csv', overwrite=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (herschelhelp_internal)",
   "language": "python",
   "name": "helpint"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}