#!/usr/bin/env python3
# ************************************************************************
# *                                                                      *
# *   Program rfc_z_from_ned.py parses the RFC catalogue, query the NED, *
# *   and returns redshifts for the RFC sources.                         *
# *                                                                      *
# * ### 08-JUL-2018 rfc_z_from_ned.py v1.0 (c) L. Petrov 11-JUL-2018 ### *
# *                                                                      *
# ************************************************************************
import pwd, sys, os, shutil, time, subprocess, datetime, signal
from urllib.parse import quote

cat_file = "/vlbi/solutions/rfc_2018b/rfc_2018b_cat.txt"
rad   = 8.0  # NED Search radius in arcsec
n_sou = 128  # max number of sources per NED requiest
url_prefix = 'http://ned.ipac.caltech.edu/cgi-bin/nnd?sr_arcsec=' + "%f" % rad + '&delimiter=bar&NO_LINKS=1&nondb=user_inplist&nondb=user_inp_sep&crosid=objname&position=ra%2Cdec&enotes=objnote&position=pretype&position=z&position=zflag&position=zrefcode&attdat=attned&gphotoms=q_value&gphotoms=q_unc&gphotoms=ned_value&gphotoms=ned_unc&diamdat=ned_maj_dia&distance=avg&distance=stddev_samp&of=ascii_bar&uplist='
wget_exe   = "/usr/bin/wget"

first_ans = '                              |(") |'  # last  line of the HTML header
last_ans  = '</PRE></TABLE>'                        # first line of the HTML footer

#
# ------------------------------------------------------------------------
#
def ners_exe ( command, verb=0 ):
    global ners_child_pid
    """
    Auxilliary routine ners_exe spawns a supborcess, executes command in 
    the context of the suborpess, waits for its completion and return 
    the completion code and results as ( returncode, out ). out is a list 
    of strings without trailing "\n". If verb >= 2, the command to be 
    executed is also printed in stdout.

    Requires  sys, os, subprocess, signal, math, datetime
    """

    if ( verb >= 2 ): 
         date_str = datetime.now().strftime("%Y.%m.%d_%H:%M:%S.%f")[0:23]
         print ( date_str, "execute:", command )
         sys.stdout.flush()
         
#
# --- Launch the subprocess
#
    proc = subprocess.Popen ( command, \
                              stdout=subprocess.PIPE, \
                              stderr=subprocess.STDOUT, \
                              shell=True )
    ners_child_pid = proc.pid
#
# --- Catch the output in list out
#
    out = []
    for line in proc.stdout:
        out.append ( str(line, encoding="utf-8").split("\n")[0] )
#
# --- wait for completion of the child process
#
    proc.wait()
    ners_child_pid = None
#
# --- Return the runcode and the output that was caught
#
    return ( proc.returncode, out )
#
# ------------------------------------------------------------------------
#

#
# --- Read contents of the RFC catalogue
#
with open(cat_file,encoding="latin") as f:
     cat = f.read().splitlines()
f.close()

url_line = ""
k = 0
coord_dict = {}
last_z_line = " "
#
# --- Cycle over the line in the RFC catalogue
#
for i in range(0,len(cat)):
    line = cat[i].strip("\n")
    if ( line[0:1] == '#' ): continue
#
# -- Extract source names
#
    ivs_name = line.split()[1]
    j2000_name = line.split()[2]
    ra_str  = line.split()[3]+ "h" + line.split()[4] + "m" + line.split()[5][:8] + "s"
    dec_str = line.split()[6]+ "d" + line.split()[7] + "m" + line.split()[8][:7] + "s"
#
# --- Build the coordinates of the RFC source in the format that NED returns.
# --- We will used it in order to match NED response with our input
#
    coord_line = line.split()[3]+ "h" + line.split()[4] + "m" + line.split()[5][:8] + "s" + " " + \
                       line.split()[6]+ "d" + line.split()[7] + "m" + line.split()[8][:4] + "..."
#
# --- Put that line in the dictionary with referencs to source names
#
    coord_dict[coord_line]={"ivs_name" : ivs_name, "j2000_name" : j2000_name}
#
# --- Append the request to the URL. NB: we web quote RA and DEC
#
    url_line = url_line + quote("\n") + quote(ra_str) + "+" + quote(dec_str)
    k = k + 1
#
# - Every n_sou run NED request or when we reached the end of the RFC catalogue
#
    if ( (k+1)%n_sou == 0 or i == len(cat) ):
#
# ------ Run NED request
# ------ when we collected request of n_sou reached the end of the catalogue
#
# ------ Just execute the huge URL and catch NED's response
#
         (ret, wget_ans ) = ners_exe ( wget_exe + " -q -O - '" + url_prefix + url_line[3:] + "'" )
         
#
# ------ Parse the NED response
#
         fl_table = False
         for line in wget_ans:
#
# ---------- This logic is for discarding html code at teh beginning of the 
# ---------- response and the end of the 
#
             if ( first_ans in line ):
                  fl_table = True
                  continue
             if ( last_ans in line ):
                  fl_table = False
                  continue
             if ( fl_table == True ) :
#
# --------------- This is the body of the repspone
#
#                  print ( line ) # %%%
                  if ( line[0:1] != " " ):
#
# -------------------- Associated the input source using the NED form of our request
#
                       ivs_name   = coord_dict[line[0:30]]["ivs_name"]
                       j2000_name = coord_dict[line[0:30]]["j2000_name"]
#
# --------------- For the redshift line
#
                  z_line = "%s %8s %s %s" % ( j2000_name, ivs_name, line[30:36], line[117:] )
                  if ( z_line[94:95] == "." and not "AbLS" in z_line ):
#
# -------------------- Good, this line has redshift and is not Absorption line system
#
                       if ( z_line[0:19] == last_z_line[0:19] ):
#
# ------------------------- If this is not the first line of our input source, mark it
#
                            z_line = z_line[0:20] + "@" + z_line[21:]
#
# -------------------- Finally, print the z-line
#
                       print ( z_line )
                       last_z_line = z_line

         url_line = ""