/**
   Generic Loader for delimited text files.

   $(LREF tabular) is the main function to be used.

   Copyright: Copyright 2013 the authors.

   License: BSD 3-Clause

   Authors: $(WEB https://github.com/agordon/ , A. Gordon), JM
*/
module nxt.tabular;

// import std.typetuple;
import std.traits: isNumeric, Select;
import std.typecons: Tuple, tuple, isTuple;
import std.functional: unaryFun;
import std.string: translate;
// import std.array;
import std.conv: text;
import std.exception: assertThrown;
import std.stdio: File;
import std.file: FileException;
import std.range;

private
{
    @safe pure void consumeDelimiter(S, D)(ref S inputString, const D delimiter)
    {
        if (inputString.empty || inputString[0] != delimiter)
	    throw new Exception("missing delimiter");

        inputString = inputString[1..$];
    }

    unittest
    {
	string s = "\t2\t3";
	consumeDelimiter(s,'\t');
	assert(s=="2\t3");
	//Trying to remove a delimiter when non is available is a throwable offense
	assertThrown!Exception(consumeDelimiter(s,'\t'));
	//Trying to remove a delimiter from an empty string is a throwable offense
	s = "";
	assertThrown!Exception(consumeDelimiter(s,' '));
    }

    @safe S consumeStringField(S,D)(ref S inputString, const D delimiter)
    {
	size_t j = inputString.length;
	foreach (i, dchar c; inputString)
	{
            if ( c == delimiter )
            {
                j = i;
                break;
            }
	}
	scope(exit) inputString = inputString[j .. $];
	return inputString[0 .. j];
    }

    unittest
    {
	// Consume the first field
	string s = "hello\tworld";
	string t = consumeStringField(s,'\t');
	assert(s=="\tworld");
	assert(t=="hello");

	// Consume the next (and last) field
	consumeDelimiter(s,'\t');
	t = consumeStringField(s,'\t');
	assert(s=="");
	assert(t=="world");

	// No string before delimiter - return an empty string
	s = "\tfoo\tbar";
	t = consumeStringField(s,'\t');
	assert(s=="\tfoo\tbar");
	assert(t=="");

	// Empty string - is a valid single (empty) field
	s = "";
	t = consumeStringField(s,'\t');
	assert(s=="");
	assert(t=="");

	// No delimiter in string - treat it as a valid single field
	s = "hello world";
	t = consumeStringField(s,'\t');
	assert(s=="");
	assert(t=="hello world");
    }

    @safe pure S quotemeta(S)(const S s)
    {
	string[dchar] meta = [ '\n' : "<LF>",
                               '\t' : "<TAB>",
                               '\r' : "<CR>",
                               '\0' : "<NULL>" ];

	return translate(s,meta);
    }

    unittest
    {
	string s="1\t2\t3\n";
	auto t = quotemeta(s);
	assert(t=="1<TAB>2<TAB>3<LF>");

	//String with null
	s="1\0002";
	t = quotemeta(s);
	assert(t=="1<NULL>2");

	//Empty string
	s="";
	t = quotemeta(s);
	assert(t=="");

	// Normal string
	s="1\\t2";
	t = quotemeta(s);
	assert(t=="1\\t2");
    }

    @safe pure string quotemeta(const char c)
    {
	string[dchar] meta = [ '\n' : "<LF>",
                               '\t' : "<TAB>",
                               '\r' : "<CR>",
                               '\0' : "<NULL>" ];
	if (c in meta)
            return meta[c];

	return [c];
    }

    unittest
    {
	assert(quotemeta('\t')=="<TAB>");
	assert(quotemeta('\r')=="<CR>");
	assert(quotemeta('\n')=="<LF>");
	assert(quotemeta('\00')=="<NULL>");
	assert(quotemeta('t')=="t");
    }

} // private


/**
   Parses string $(D input), delimited by character $(D delimiter), into a tuple of variables $(arg).

   Returns:
   On success, the function returns nothing (void), and all the members of the tuple are populated.

   Throws:
   $(XREF std.exception.Exception) on failure to correctly parse the string.

   Example:
   ----
   string s = "Hello World 42";
   Tuple!(string,string,int) t;
   parseDelimited(s,' ',t);
   assert(t[0]=="Hello");
   assert(t[1]=="World");
   assert(t[2]==42);
   ----

   Notes:
   $(OL
   $(LI Parsing is much stricter (and less tolerant) than $(XREF std.format.formattedRead))
   $(LI White-space is never automatically skipped)
   $(LI A space delimiter consume only space character (ASCII 20), not TAB (ASCII 9))
   $(LI Multiple consecutive delimiters are not consumed as one delimiter (e.g. "1\t\t\t2" is considerd a string with four fields - it has three delimiters. It will throw an exception because empty fields are not allowed).)
   $(LI All fields must exist (i.e. if the tuple $(D arg) has 3 members, the $(D input) string must contain two delimiters and three valid values))
   $(LI For a string field, empty values are not acceptable, will throw an exception)
   $(LI Extra characters at the end of a field or the line will throw an exception)
   )

*/
@safe void parseDelimited(Data)(const string input,
                                const char delimiter,
                                ref Data arg)
{
    string remainingInput = input;

    foreach (i, T; Data.Types)
    {
        //TODO: Handle other types (for now, only numeric or strings)
        static if (isNumeric!T)
        {
            try
            {
                // consume a numeric field
                static import std.conv;
                arg[i] = std.conv.parse!T(remainingInput);
            }
            catch ( std.conv.ConvException e )
            {
                throw new Exception(text("failed to parse numeric value in field ", i+1,
                                         " (text is '",quotemeta(remainingInput),"')"));
            }
        }
        else
 	{
            // consume a string field
            arg[i] = consumeStringField(remainingInput,delimiter);
            if (arg[i].empty)
                throw new Exception(text("empty text at field ", i+1,
                                         " (remaining text is '",quotemeta(remainingInput),"')"));
        }

        static if (i<Data.length-1)
        {
            //Not the last field - require more input
            if (remainingInput.empty)
                throw new Exception(text("input terminated too soon (expecting ",
                                         Data.length," fields, got ", i+1, ")"));

            //Following the converted value of this field,
            //require a delimiter (to prevent extra characters, even whitespace)
            if (remainingInput[0] != delimiter)
                throw new Exception(text("extra characters in field ",i+1,
                                         " (starting at '",quotemeta(remainingInput),"')"));
            consumeDelimiter(remainingInput,delimiter);
        }
        else
        {
            // Last field: check for extra input
            if (!remainingInput.empty)
                throw new Exception(text("extra characters in last field ",i+1,
                                         " (starting at '",quotemeta(remainingInput),"')"));
        }

    }
}

unittest
{
    Tuple!(int,string,int) a;
    parseDelimited("1 2 3",' ',a);
    assert(a[0]==1 && a[1]=="2" && a[2]==3);

    parseDelimited("1\t2\t3",'\t',a);
    assert(a[0]==1 && a[1]=="2" && a[2]==3);

    //Extra delimiter at the end of the line is not OK
    assertThrown!Exception(parseDelimited("1 2 3 ",' ',a));

    //Invalid number on first field (parse!int should fail)
    assertThrown!Exception(parseDelimited(".1 2 3",' ',a));

    //Extra characters in field 1 (After successfull parse!int)
    assertThrown!Exception(parseDelimited("1. 2 3",' ',a));

    //Line contains too many fields
    assertThrown!Exception(parseDelimited("1 2 3 4",' ',a));

    //Line is too short
    assertThrown!Exception(parseDelimited("1 2",' ',a));

    //non-space/tab delimiter is fine
    parseDelimited("1|2|3",'|',a);
    assert(a[0]==1 && a[1]=="2" && a[2]==3);
    parseDelimited("1|  2  |3",'|',a);
    assert(a[0]==1 && a[1]=="  2  " && a[2]==3);

    //Spaces are bad (and not ignored) if delimiter is not space (for numeric fields)
    assertThrown!Exception(parseDelimited("1 |2|3",'|',a));
    assertThrown!Exception(parseDelimited(" 1|2|3",'|',a));
    assertThrown!Exception(parseDelimited(" 1|2| 3",'|',a));
    assertThrown!Exception(parseDelimited("1|2|3 ",'|',a));

    //For string fields, empty values are not OK (different from formattedRead())
    assertThrown!Exception(parseDelimited("1||3",'|',a));

    //For string fields, last value can't be empty (different from formattedRead())
    Tuple!(int,string,string) b;
    assertThrown!Exception(parseDelimited("1|2|",'|',b));

    //One field is OK
    Tuple!(string) c;
    parseDelimited("foo",' ',c);
    assert(c[0]=="foo");

    //Fields that are OK for floating-point types should not work for integers (extra characters)
    Tuple!(real,int) d;
    parseDelimited("4.5 9",' ',d);
    assert(d[0]==4.5 && d[1]==9);
    Tuple!(int,real) e;
    assertThrown!Exception(parseDelimited("4.5 9",' ',e));

    //scientific notation - OK for floating-point types
    Tuple!(double,double) f;
    parseDelimited("-0.004e3 +4.3e10",' ',f);
    assert(f[0]==-0.004e3 && f[1]==43e9);

    //Scientific notation - fails for integars
    Tuple!(int,int) g;
    assertThrown!Exception(parseDelimited("-0.004e3 +4.3e10",' ',g));
}


/**
   Loads a delimited text file, line-by-line, parses the line into fields, and calls a delegate/function for each line.

   Returns:
   On success, the function returns nothing (void), the call back function have been called for every line.

   Throws:
   $(XREF std.exception.Exception) on failure to correctly parse a line.
   $(XREF std.file.FileException) on I/O failures.

   Example:
   ----
// Load a text file with three numeric columns,
// Store the tuple in an array
// (NOTE: this is a naive, inefficient way to populate an array, see NOTES)
alias Tuple!(int,int,int) T;
T[] t;
tabular!( T,           // The number and types of the (expected) fields in the file
delegate(x)
{ t ~= x; }, // for each line read, call this function. X will be of type T.
'\t'         // The delimiter (default = TAB)
)("file.txt"); // The file name to read.
----

Example:
----
// Load a text file with three numeric columns,
// Use the second column as a KEY and the third column as the VALUE.
alias Tuple!(int,int,int) T;
int[int] data;
tabular!( T,              // The number and types of the (expected) fields in the file
delegate(x)
{   // for each line read, call this function. X will be of type T.
data[x[1]] = x[2] ;
},
'\t'             // The delimiter (default = TAB)
)("file.txt");    // The file name to read.
----

Notes:
$(OL
$(LI See $(LREF parseDelimited) for details about parsing the delimited lines of the fiile)
$(LO
)

TODO: Make this an InputRange

*/
void tabular(Members, alias storeFunction, char delimiter='\t')(const string filename)
{
    static assert (isTuple!Members,"tabular: 1st template parameter must be a Tuple with the expected columns in the file");

    auto f = File(filename);
    scope(exit) f.close();
    auto lines=0;

    alias unaryFun!storeFunction _Fun;
    Members data;

    import nxt.bylinefast: byLineFast;
    foreach (origline; f.byLineFast())
    {
        ++lines;
        string line = origline.idup;
        try
        {
            parseDelimited(line, delimiter, data);
            _Fun(data);
        }
        catch ( Exception e )
        {
            throw new FileException(filename,text("invalid input at line ", lines,
                                                  ": expected ", data.tupleof.length,
                                                  " fields ",typeof(data.tupleof).stringof,
                                                  " delimiter by '",quotemeta(delimiter),
                                                  "' got '", origline,
                                                  "' error details: ", e.msg ));
        }
    }
}

unittest
{
    import std.file ;
    auto deleteme = testFilename();
    write(deleteme,"1 2 3\n4 5 6\n");
    scope(exit)
    { assert(exists(deleteme)); remove(deleteme); }

    //Load a text file, with three fields, delimiter with spaces.
    alias Tuple!(int,int,int) T;
    T[] t;
    tabular!( T,         // The number and types of the (expected) fields in the file
             delegate(x)
             { t ~= x; }, // for each line read, call this function. X will be of type T.
             ' '        // The delimiter (default = TAB)
        )(deleteme); // The file name to read.
	assert(t.length==2);
	assert(t[0] == tuple(1,2,3));
	assert(t[1] == tuple(4,5,6));

	//Any kind of invalid data should throw an exception
	//NOTE: the delegate function does nothing, because we don't care about the data
	//      in this test.
	//NOTE: see more test cases for failed parsing in the unittest of 'parseDelimited'.
	auto deleteme2 = testFilename() ~ ".2";
	write(deleteme2,"1 Foo 3\n4 5 6\n"); // conversion will fail in the first line
	scope(exit)
        { assert(exists(deleteme2)); remove(deleteme2); }
	assertThrown!Exception( tabular!( T, (x) => {}, ' ')(deleteme2)) ;
}

/**
Loads a delimited text file, line-by-line, parses the line into fields, returns an array of fields.

Returns:
On success, returns an array of tuples, based on template parameters.

Throws:
$(XREF std.exception.Exception) on failure to correctly parse a line.
$(XREF std.file.FileException) on I/O failures.

Example:
----
// Load a text file, tab-delimited, with three numeric columns.

auto data = tabularArray!('\t', int,int,int)("file.txt");

// data[0] will be of type Tuple!(int,int,int)
----
*/
Select!(Types.length == 1, Types[0][], Tuple!(Types)[])
tabularArray(char delimiter, Types...)(string filename)
{
    alias RetT = typeof(return);

    RetT result;
    Appender!RetT app;
    alias Members = ElementType!RetT;

    tabular! ( Members, x => app.put(x) , delimiter ) (filename);

    return app.data;
}

unittest
{
    import std.file ;
    auto deleteme = testFilename() ~ ".3";
    write(deleteme,"1 2 3\n4 5 6\n");
    scope(exit)
    { assert(exists(deleteme)); remove(deleteme); }

    //Load a text file, with three fields, delimiter with spaces.
    auto t = tabularArray!( ' ', // delimiter
                            int, int, int // expected fields in the text file
        )(deleteme);
    assert(t.length==2);
    assert(t[0] == tuple(1,2,3));
    assert(t[1] == tuple(4,5,6));
}

version(unittest) string testFilename(string file = __FILE__, size_t line = __LINE__)
{
	import std.path;
	import std.process: thisProcessID;
	return text("deleteme-.", thisProcessID(), ".", baseName(file), ".", line);
}

/*
On Thursday, 16 May 2013 at 10:35:12 UTC, Dicebot wrote:
> Want to bring into discussion people that are not on Google+.
> Samuel recently has posted there some simple experiments with
> bioinformatics and bad performance of Phobos-based snippet has
> surprised me.
>
> I did explore issue a bit and reported results in a blog post
> (snippets are really small and simple) :
> http://dicebot.blogspot.com/2013/05/short-performance-tuning-story.html
>
> One open question remains though - can D/Phobos do better here?
> Can some changes be done to Phobos functions in question to
> improve performance or creating bioinformatics-specialized
> library is only practical solution?

I bet the problem is in readln. Currently, File.byLine() and
readln() are extremely slow, because they call fgetc() one char
at a time.

I made an "byLineFast" implementation some time ago that is 10x
faster than std.stdio.byLine. It reads lines through rawRead, and
using buffers instead of char by char.

I don't have the time to make it phobos-ready (unicode, etc.).
But I'll paste it here for any one to use (it works perfectly).

--jm
*/