#!/usr/bin/env python3
# ************************************************************************
# *                                                                      *
# *   Program rfc_z_from_ned.py parses the RFC catalogue, query the NED, *
# *   and returns AGN type for the RFC sources.                          *
# *                                                                      *
# * ### 08-JUL-2018 rfc_t_from_ned.py v2.2 (c) L. Petrov 02-FEB-2019 ### *
# *                                                                      *
# ************************************************************************
import pwd, sys, os, re, shutil, time, subprocess, datetime, operator, signal, math
from urllib.parse import quote

prev_cat_file = "/vlbi/solutions/rfc_2019a/rfc_2019a_cat.txt"
cat_file      = "/vlbi/solutions/rfc_2019b/rfc_2019b_cat.txt"

inp_t_file= "/tmp/empty.txt"
rad   = 4.0
n_sou = 128

url_prefix = 'https://ned.ipac.caltech.edu/cgi-bin/nnd?sr_arcsec=' + "%f" % rad + '&delimiter=bar&NO_LINKS=1&nondb=user_inp_msg&nondb=user_inplist&nondb=user_inp_sep&crosid=objname&position=ra%2Cdec&position=pretype&gadata=morphol&attdat_CON=S&attdat=attned&gphotoms=q_value&gphotoms=q_unc&gphotoms=ned_value&gphotoms=ned_unc&diamdat=ned_maj_dia&distance=avg&distance=stddev_samp&uplist='

with open(prev_cat_file,encoding="latin") as f:
     pre = f.read().splitlines()
f.close()

with open(cat_file,encoding="latin") as f:
     cat = f.read().splitlines()
f.close()

with open(inp_t_file,encoding="latin") as f:
     inp_t = f.read().splitlines()
f.close()

first_ans = '<PRE><strong> | Input Position / Object Name |'
last_ans  = '</PRE></TABLE>'

#
# ------------------------------------------------------------------------
#
def ners_exe ( command, verb=0 ):
    global ners_child_pid
    """
    Auxilliary routine ners_exe spawns a supborcess, executes command in 
    the context of the suborpess, waits for its completion and return 
    the completion code and results as ( returncode, out ). out is a list 
    of strings without trailing "\n". If verb >= 2, the command to be 
    executed is also printed in stdout.

    Requires  sys, os, subprocess, signal, math, datetime
    """

    if ( verb >= 2 ): 
         date_str = datetime.now().strftime("%Y.%m.%d_%H:%M:%S.%f")[0:23]
         print ( date_str, "execute:", command )
         sys.stdout.flush()
         
#
# --- Launch the subprocess
#
    proc = subprocess.Popen ( command, \
                              stdout=subprocess.PIPE, \
                              stderr=subprocess.STDOUT, \
                              shell=True )
    ners_child_pid = proc.pid
#
# --- Catch the output in list out
#
    out = []
    for line in proc.stdout:
        out.append ( str(line, encoding="utf-8").split("\n")[0] )
#
# --- wait for completion of the child process
#
    proc.wait()
    ners_child_pid = None
#
# --- Return the runcode and the output that was caught
#
    return ( proc.returncode, out )

#
# ------------------------------------------------------------------------
#

url_line = ""
k = 0
coord_dict = {}
last_t_line = " "
#
# --- Cycle over RFC catalogue
#
for i in range(0,len(cat)):
    line = cat[i].strip("\n")
    if ( line[0:1] == '#' ): continue
    fl_found_in_prev = 0
    for lin in pre:
        if ( lin[12:22] == line[12:22] ): fl_found_in_prev  = 1
#
# -- Extract source names
#
    ivs_name = line.split()[1]
    j2000_name = line.split()[2]
#
# --- Check whether this source is in the output z-file
#
    fl_found_in_t = 0
    for lin in inp_t:
        if ( lin[0:10] == j2000_name ):
             fl_found_in_t = 1
#
    ra_str  = line.split()[3]+ "h" + line.split()[4] + "m" + line.split()[5][:8] + "s"
    dec_str = line.split()[6]+ "d" + line.split()[7] + "m" + line.split()[8][:7] + "s"
#
# --- Build the coordinate name that NED returns
#
    coord_line = line.split()[3]+ "h" + line.split()[4] + "m" + line.split()[5][:8] + "s" + " " + \
                       line.split()[6]+ "d" + line.split()[7] + "m" + line.split()[8][:4] + "..."
#
# --- Put that line in the dictionary with referencs to source names
#
    coord_dict[coord_line]={"ivs_name" : ivs_name, "j2000_name" : j2000_name}
    if ( fl_found_in_prev == 0 and fl_found_in_t == 0 ): 
#         print ( "Used_sou: ", line[12:22], flush=True ) # %%%
         url_line = url_line + quote("\n") + quote(ra_str) + "+" + quote(dec_str)
         k = k + 1
#
    if ( k > 0 and len(url_line) > 0 and ( (k+1)%n_sou == 0 or i == len(cat)-1 ) ):
#
# ------ Run NED request
# ------ when we collected request of n_sou reached the end of the catalogue
#
#         print ( "++++++++++++++++++++++++++ ", ( i, k, n_sou, len(cat) ) ) # %%%
#         print ( "com= " + "/usr/bin/wget -q -O - '" + url_prefix + url_line[3:] + "'"  ) # %%%%
         (ret, wget_ans ) = ners_exe ( "/usr/bin/wget -q -O - '" + url_prefix + url_line[3:] + "'" )
         
#
# ------ Parse the NED response
#
         fl_table = False
         for line in wget_ans:
             if ( first_ans in line ):
                  fl_table = True
                  continue
             if ( last_ans in line ):
                  fl_table = False
                  continue
             if ( fl_table == True ) :
#                  print ( "line= ", line ) # %%%%%%%%%%%%%%%%
                  if ( "WARNING:" in line or "WARNING " in line ): continue
                  if ( "NOTE:"    in line or "NOTE "    in line ): continue
                  if ( "ERROR:"   in line or "ERROR "   in line ): continue
#                  print ( "UUU line[3:4 >>" + line[3:4] + "<<" ) # %%%%%%%%%%%%%%%%
                  if ( line[3:4] != " " ):
                       ivs_name   = coord_dict[line[2:32]]["ivs_name"]
                       j2000_name = coord_dict[line[2:32]]["j2000_name"]
#                       print ( "TTT  line= ", line[2:32], ' names= ', ( ivs_name, j2000_name ) ) # %%%%%%%%%%%%%%%%
                  t_line = "%s %-8s %s" % ( j2000_name, ivs_name, line[32:102] )
                  line = line.lower()
                  gal_type = None
                  if ( "bllac" in line or "bl lac" in line ):
                        t_line = t_line[0:83] + "BLLac |"

                  if ( "candidate" in line and "qso" in line ):
                        t_line = t_line[0:83] + "QSOc  |"

                  if ( "Q_Lens" in line ):
                        t_line = t_line[0:83] + "QSO   |"

                  if ( t_line[83:89] == "RadioS" or \
                       t_line[83:89] == "VisS  " or \
                       t_line[83:89] == "*     " or \
                       t_line[83:89] == "G     "    ):
                       if ( "candidate" in line and "blazar" in line ):
                            t_line = t_line[0:83] + "BZc   |"
                       elif ( "blazar" in line ):
                            t_line = t_line[0:83] + "BZ    |"


                  if ( "sy" in line and not "sym" in line ):
                       ind = line.index ( "sy" )
                       if ( line[ind+2:ind+3] == " " ):
                            if ( len(line[ind+3:].split()) > 1 ):
                                 gal_type = "sy" + line[ind+3:].split()[0]
                            else:
                                 gal_type = "sy"
                       else:
                            gal_type = line[ind:].split()[0]

                       if ( "|" in gal_type ):
                            gal_type = gal_type[0:gal_type.index("|")]
                       if ( gal_type == "s" ): gal_type = "sy"

                  if ( "liner" in line ):
                        gal_type = "liner"

                  if ( gal_type ):
                       gal_type = gal_type[0:1].upper() + gal_type[1:]
                       t_line = t_line + " " + gal_type
                        
                  if ( t_line[0:19] == last_t_line[0:19] ):
                       t_line = t_line[0:20] + "@" + t_line[21:]
                  print ( t_line )
                  last_t_line = t_line

         url_line = ""