Re: [gnugo-devel] reading connections

[Gunnar Farneback]

Tristan wrote:
> I thought that the following gtp command is also useful :
> 
> non_transitivity A5 B8 C10
> #? [1 B7]

I'm sure it is, but what does it do? Is there any support in
readconnect.c yet to implement it?

> and is there a command to track down the time used by the engine
> for each problem in the test suite ?

No, but it's possible to write a program which feeds the gtp commands
one at a time and measures the time spent on each. Appended is a pike
script I've written which gives quite a bit more information than the
standard regression scripts do, including timing info. To run it you
need to have a pike interpreter installed, available from
http://pike.roxen.com. Give it a test suite file as parameter. Sorry,
no documentation provided, but feel free to ask if something is
unclear. 

/Gunnar

#!/usr/bin/env pike

static inherit Thread.Queue:write_queue;
static int debug = 0;
static object machine;
static mapping(int:string) correct_results = ([]);
static multiset expected_failures = (<>);
static int timebase;
static float last_time;
static float total_time = 0.0;
static int total_pass = 0;
static int total_fail = 0;

void Send(string s)
{
    if (debug)
        werror("Sent: " + s + "\n");
    write_queue::write(s + "\n");
}

static void Finish(array(int) total_nodes, array(int) unexpected_failures,
                   array(int) unexpected_passes)
{
    werror("Total nodes: %d %d %d\n", total_nodes[0], total_nodes[1],
           total_nodes[2]);
    werror("Total time: %4.2f\n", total_time);
    werror("Number of passed tests: %d/%d\n", total_pass,
           total_pass + total_fail);

    if (sizeof(unexpected_failures) > 0)
        werror("%d unexpected failure%s (%s)\n", sizeof(unexpected_failures),
               sizeof(unexpected_failures) != 1 ? "s" : "",
               ((array(string)) unexpected_failures) * ", ");

    if (sizeof(unexpected_passes) > 0)
        werror("%d unexpected pass%s (%s)\n", sizeof(unexpected_passes),
               sizeof(unexpected_passes) != 1 ? "es" : "",
               ((array(string)) unexpected_passes) * ", ");

    exit(0);
}

static void ProgramReader(object f)
{
    int move;
    string stored = "";
    array(int) nodes = ({0, 0, 0});
    array(int) total_nodes = ({0, 0, 0});
    array(int) unexpected_failures = ({});
    array(int) unexpected_passes = ({});
    int test_number;
    while (1)
    {
        string s = f->gets();
        float current_time = time(timebase);
        if (!s)
            break;
        if (debug)
            werror("Recv: " + s + "\n");
        
        int number;
        string answer;
        if (sscanf(s, "=%d %s", number, answer)) {
            if (number < 10000) {
                test_number = (int) number;
                string correct = correct_results[test_number];
                int negate = 0;
                if (correct[0] == '!')
                {
                    correct = correct[1..];
                    negate = 1;
                }
                object re = Regexp(correct);
                string result = (negate ^ re->match(answer)) ? "pass" : "fail";

                if (result == "pass")
                    total_pass++;
                else
                    total_fail++;

                if (result == "pass" && expected_failures[test_number])
                {
                    result = "PASS";
                    unexpected_passes += ({test_number});
                }
                if (result == "fail" && !expected_failures[test_number])
                {
                    result = "FAIL";
                    unexpected_failures += ({test_number});
                }
                stored += sprintf("%5.2f - %s %s (%s)",
                                  current_time - last_time,
                                  result,
                                  answer,
                                  correct_results[test_number]);
                total_time += current_time - last_time;
                last_time = current_time;
            }
            else
            {
                nodes[number-10000] = (int) answer;
                total_nodes[number - 10000] += (int) answer;
            }
            if (number == 10002) {
                werror("%4d %8d %5d %7d   %s\n", test_number,
                       nodes[0], nodes[1], nodes[2], stored);
                stored = "";
            }
        }
    }
    f->close();
    if (debug)
        werror("Reader closed down.\n");
    Finish(total_nodes, unexpected_failures, unexpected_passes);
}

static void ProgramWriter(object f)
{
    while (1)
    {
        string s = write_queue::read();
        if (s == "")
            break;
        f->write(s);
    }
    f->close();
    if (debug)
        werror("Writer closed down\n");
}

int main(int argc, array(string) argv)
{
    array(string) program_start_array = ({"../interface/gnugo", "--quiet",
                                          "--mode", "gtp"});
    object f1 = Stdio.File();
    object pipe1 = f1->pipe();
    object f2 = Stdio.FILE();
    object pipe2 = f2->pipe();
    machine = Process.create_process(program_start_array + argv[2..],
                                     (["stdin":pipe1, "stdout":pipe2]));
    thread_create(ProgramReader, f2);
    thread_create(ProgramWriter, f1);

    string testsuite = Stdio.read_bytes(argv[1]);
    if (!testsuite) {
      werror("Couldn't find " + argv[1] + "\n");
      exit(1);
    }
    int number;
    string answer;
    string expected;

    timebase = time();
    last_time = time(timebase);

    foreach (testsuite/"\n", string s) {
        if (sscanf(s, "%d", number) == 1) {
            if (number >= 10000)
                continue;
            Send("reset_reading_node_counter");
            Send("reset_owl_node_counter");
            Send("reset_life_node_counter");
            Send(s);
            Send("10000 get_reading_node_counter");
            Send("10001 get_owl_node_counter");
            Send("10002 get_life_node_counter");
        }
        else if (sscanf(s, "#? [%[^]]]%s", answer, expected))
        {
            correct_results[(int)number] = answer;
            if (expected == "*")
                expected_failures[(int)number] = 1;
        }
        else
            Send(s);
    }
    
    Send("quit");
    Send("");
    return -1;
}