#!/usr/bin/env slsh
require ("curl");
require ("pcre");
require ("cmdopt");

private variable Version = "0.3.2";

private variable Dir_Regexp = pcre_compile ("^[dl][-+rwxtsr]+ .* ([-0-9A-Z_a-z./]+)$");
private variable File_Regexp = pcre_compile ("^-[-rwxtsr+]+ .* ([-0-9A-Z_a-z.]+)$");

private define write_to_string (sp, data)
{
   @sp = strcat (@sp, data);
   return 1;
}

private define write_to_fp (fp, data)
{
   variable len = bstrlen (data);
   if (bstrlen(data) != fwrite (data, fp))
     return -1;
   return 0;
}

private define get_dir_listing (c, dir)
{
   variable data = "";
   curl_setopt (c, CURLOPT_URL, path_concat (dir, ""));
   curl_setopt (c, CURLOPT_WRITEFUNCTION, &write_to_string, &data);
   variable e;
   try (e)
     {
	curl_perform (c);
     }
   catch CurlError:
     {
	() = fprintf (stderr, "Curl Error: %S\n", e.message);

	if (e.error == CURLE_LDAP_SEARCH_FAILED)
	  () = fprintf (stderr, "FTP directory lookup failed: %S\n", dir);
	else
	  () = fprintf (stderr, "The server may be down.  Please try again later\n");

	exit (1);
     }

   variable lines = strtok (data, "\r\n");
   variable dir_list = {};
   variable file_list = {};
   foreach (lines)
     {
	variable line = ();
	if (pcre_exec (Dir_Regexp, line))
	  {
	     list_append (dir_list, pcre_nth_substr (Dir_Regexp, line, 1));
	     continue;
	  }
	if (pcre_exec (File_Regexp, line))
	  {
	     list_append (file_list, pcre_nth_substr (File_Regexp, line, 1));
	     continue;
	  }
     }
   return dir_list, file_list;
}

private define find_obsid_url (c, rooturl, obsid);
private define find_obsid_url (c, rooturl, obsid)
{
   variable dirlist, filelist;
   (dirlist,) = get_dir_listing (c, rooturl);

   % If we are searching .../cat/, then don't recurse more.
   variable this_dir = path_basename (path_dirname (path_concat (rooturl,"")));
   variable recurse_ok = strncmp (this_dir, "cat", 3);

   foreach (dirlist)
     {
	variable subdir = ();
	variable dir = path_concat (rooturl, subdir);

	%() = fprintf (stdout, "%s\n", dir);

	if (subdir == obsid)
	  return dir;

	!if (recurse_ok)
	  continue;

	dir = find_obsid_url (c, dir, obsid);
	if (dir != NULL)
	  return dir;
     }
   return NULL;
}

private define get_recursive_listing (c, rooturl);
private define get_recursive_listing (c, rooturl, list)
{
   variable dirlist, filelist;
   (dirlist,filelist) = get_dir_listing (c, rooturl);

   foreach (filelist)
     {
	variable file = ();
	list_append (list, path_concat (rooturl, file));
     }

   foreach (dirlist)
     {
	variable subdir = ();
	get_recursive_listing (c, path_concat (rooturl, subdir), list);
     }
}

private define open_output_file (file)
{
   variable fp = fopen (file, "wb");
   if (fp != NULL)
     return fp;

   () = system (sprintf ("mkdir -p %s", path_dirname (file)));

   return fopen (file, "wb");
}

private define usage ()
{
   variable pgm = path_basename (__argv[0]);
   variable help = 
     ["Version $Version Usage: $pgm [options] OBSID"$,
      "Options:",
      "  -h|--help                   This help",
      "  -i|--info                   Show files that would be downloaded",
      %"  -c|--cat science|cal|er     Catalogues to search (default is all)",
      "  -o|--dir output-dir         Output directory (default obs_OBSID)",
      "  --download[=type-list]      Types of files to download",
      "  --show-urls                 Display files as URLs but do not download",
      "  -v|--verbose                Show what is happening",
      "",
      "Examples:",
      "Download JED's standard set of files for obs 105:",
      " $pgm --download 105"$,
      "",
      "Show what files would be downloaded from JED's standard set:",
      " $pgm --info --download 105"$,
      "",
      "Download the bias and pbk files for obs 105:",
      " $pgm --download=bias0,pbk0 105"$,
      "",
      "Download all files for obs 105:",
      " $pgm --download=all 105"$,
      " $pgm 105"$,
      "",
      "Download calibration obsid 61273 to cal/61273:",
      " $pgm --download --dir=cal/61273 61273"$,

      ""];

   foreach help (help)
     () = fprintf (stderr, "%s\n", help);
   exit (1);
}

private define error_routine (msg)
{
   () = fprintf (stderr, "%s\n", msg);
   usage ();
}

define slsh_main ()
{
   variable info_mode = 0;
   variable jed_set = "evt1,msk1,flt1,bpix1,stat1,asol1,mtl1,pbk0,bias0,dtf1";
   variable set = NULL;
   variable catalogues = "science,cal,er";
   variable output_dir = NULL;
   variable verbose = 0;
   variable show_urls = 0;

   variable opts = cmdopt_new (&error_routine);
   opts.add ("h|help", &usage);
   opts.add ("show-urls", &show_urls);
   opts.add ("v|verbose", &verbose);
   opts.add ("i|info", &info_mode);
   opts.add ("download", &set; type="string", optional=jed_set);
   opts.add ("c|cat", &catalogues; type="string");
   opts.add ("o|dir", &output_dir; type="string");

   variable i = opts.process (__argv, 1);
   if (i + 1 != __argc)
     usage (NULL);

   variable obsid = __argv[i];

   if (set != NULL)
     {
	set = strlow (set);
	if (set == "all")
	  set = NULL;
     }

   variable host = "cda.harvard.edu";
   variable c, url, cat;

#iffalse
   foreach cat (strtok (catalogues, ","))
     {
	url = "ftp://$host/pub/$cat/"$;
	vmessage ("Trying %s", url);
	c = curl_new (url);

	%curl_setopt (c, CURLOPT_VERBOSE);
	%curl_setopt (c, CURLOPT_FTP_USE_EPSV, 0);
	url = find_obsid_url (c, url, obsid);
	if (url != NULL)
	  break;
     }
#else
   url = sprintf ("ftp://%s/pub/byobsid/%c/%s", host, obsid[-1], obsid);
   c = curl_new (url);
#endif

   if (url == NULL)
     {
	() = fprintf (stderr, "Unable to find the directory for obsid %s\n", obsid);
	exit (1);
     }

   variable files = {};
   get_recursive_listing (c, url, files);
   
   variable regexp = NULL;
   if (set != NULL)
     {
	regexp = sprintf (".*_(%s)\\.fits(\\.gz)?$", strtrans (strcompress (set, ","), ",", "|"));
	regexp = pcre_compile (regexp);
     }
   url = path_concat (url, "");   %  make sure final / is there
   variable len = strlen (url);

   if ((info_mode == 0) and (show_urls == 0))
     {
	if (output_dir == NULL)
	  output_dir = "obs_${obsid}"$;
	() = mkdir (output_dir);
     }
   
   foreach (files)
     {
	variable file = ();
	variable filename = substr (file, len+1, -1);

	if (regexp != NULL)
	  {
	     if (0 == pcre_exec (regexp, filename))
	       continue;
	  }
	if (show_urls)
	  {
	     () = fprintf (stdout, "%s\n", file);
	     continue;
	  }

	if (verbose || info_mode) 
	  () = fprintf (stdout, "Downloading %s\n", file);

	if (info_mode)
	  continue;
	
	filename = path_concat (output_dir, filename);
	variable fp = open_output_file (filename);
	if (fp == NULL)
	  {
	     () = fprintf (stderr, "Unable to open %s\n", filename);
	     exit (1);
	  }
	curl_setopt (c, CURLOPT_WRITEFUNCTION, &write_to_fp, fp);
	curl_setopt (c, CURLOPT_URL, file);
	curl_perform (c);
	if (-1 == fclose (fp))
	  {
	     () = fprintf (stderr, "Unable to close %s -- disk full?\n", filename);
	     exit (1);
	  }
     }
}
