AParser.pm - Perl модуль управления A-Parser через API

29 июн 2015
  • AParser.pm - Perl модуль, реализует текущее API
    История изменений:
    • 07.03.2014 - добавлен метод moveTask
    • 11.02.2015 - добавлены методы getTaskResultsFile, deleteTaskResultsFile

    Скачать: AParser.pm
    Пример использования:
    Код:
    use AParser;
    use Data::Dumper;
    
    my $parser = AParser->new('http://127.0.0.1:9091/API', 'password');
    
    warn "Ping result:\n", Dumper $parser->ping();
    warn "Info result:\n", Dumper $parser->info();
    warn 'Total proxies: ', scalar keys %{ $parser->getProxies() }, "\n";
    
    warn "getParserPreset result:\n", Dumper $parser->getParserPreset('SE::Google', 'default');
    
    warn "oneRequest result:\n", Dumper $parser->oneRequest('SE::Google::PR', 'default', 'a-parser.com', 'doLog' => 0, 'rawResults' => 1, 'options' => [
        {'type' => 'override', 'id' => 'proxyretries', 'value' => 3}
    ]);
    warn "bulkRequest result:\n", Dumper $parser->bulkRequest('SE::Google::PR', 'default', 10, ['a-parser.com', 'google.com', 'yandex.ru'], 'doLog' => 0, 'rawResults' => 1);
    
    my($taskUid, $error) = $parser->addTask('default', 'default', 'text', ['google.com', 'yandex.com'],
        'parsers' => [['SE::Google::PR', 'default']], #override default SE::Google parser
    );
    
    if($taskUid) {
        warn "Add task: ", $taskUid, "\n";
        warn "Move task: ", $parser->moveTask($taskUid, $up), "\n";
        $parser->waitForTask($taskUid);
        warn "Task state:\n", Dumper $parser->getTaskState($taskUid);
        warn 'File name: ', $parser->getTaskConf($taskUid)->{'resultsFileName'};
        warn 'Download url: ', $parser->getTaskResultsFile($taskUid), "\n";
        warn 'Delete results file: ', $parser->deleteTaskResultsFile($taskUid), "\n";
        warn "Delete task: ", $parser->changeTaskStatus($taskUid, 'deleting'), "\n";
    }
    else {
        warn $error
    };
    

    Результат:
    Код:
    Ping result:
    $VAR1 = 'pong';
    Info result:
    $VAR1 = {
              'tasksInQueue' => 11,
              'pid' => '7308',
              'version' => '1.1.197'
            };
    Total proxies: 3
    getParserPreset result:
    $VAR1 = {
              'queryformat' => '$query',
              'parsenotfound' => 1,
              'gl' => '',
              'pagecount' => 5,
              'do_gzip' => 1,
              'domain' => 'www.google.com',
              'timeout' => 60,
              'useproxy' => 1,
              'antigatepreset' => 'default',
              'extraquery' => '',
              'serptime' => '',
              'location' => '',
              'usesessions' => 1,
              'filter' => 1,
              'linksperpage' => 100,
              'serp' => '',
              'useantigate' => 0,
              'proxyretries' => 10,
              'requestdelay' => 0,
              'proxybannedcleanup' => 300,
              'rawdata' => 0,
              'formatresult' => '$serp.format(\'$link\\n\')',
              'lr' => '',
              'max_size' => 204800
            };
    oneRequest result:
    $VAR1 = {
              'logs' => [],
              'results' => [
                             {
                               'info' => {
                                           'success' => 1,
                                           'retries' => 2,
                                           'stats' => {
                                                        'success' => 1,
                                                        'retries' => 2,
                                                        'proxiesUsed' => 2,
                                                        'requests' => 2,
                                                        'queries' => 1
                                                      }
                                         },
                               'query' => {
                                            'first' => 'a-parser.com',
                                            'lvl' => 0,
                                            'query' => 'http://a-parser.com',
                                            'queryUid' => '0',
                                            'orig' => 'a-parser.com'
                                          },
                               'pr' => '0',
                               'data' => 'none',
                               'resultsCount' => 1
                             }
                           ]
            };
    bulkRequest result:
    $VAR1 = {
              'logs' => [],
              'results' => [
                             {
                               'info' => {
                                           'success' => 1,
                                           'retries' => 1,
                                           'stats' => {
                                                        'success' => 1,
                                                        'retries' => 1,
                                                        'proxiesUsed' => 1,
                                                        'requests' => 1,
                                                        'queries' => 1
                                                      }
                                         },
                               'query' => {
                                            'first' => 'a-parser.com',
                                            'lvl' => 0,
                                            'query' => 'http://a-parser.com',
                                            'queryUid' => '0',
                                            'orig' => 'a-parser.com'
                                          },
                               'pr' => '0',
                               'data' => 'none',
                               'resultsCount' => 1
                             },
                             {
                               'info' => {
                                           'success' => 1,
                                           'retries' => 1,
                                           'stats' => {
                                                        'success' => 1,
                                                        'retries' => 1,
                                                        'proxiesUsed' => 1,
                                                        'requests' => 1,
                                                        'queries' => 1
                                                      }
                                         },
                               'query' => {
                                            'first' => 'google.com',
                                            'lvl' => 0,
                                            'query' => 'http://google.com',
                                            'queryUid' => '1',
                                            'orig' => 'google.com'
                                          },
                               'pr' => '9',
                               'data' => 'none',
                               'resultsCount' => 1
                             },
                             {
                               'info' => {
                                           'success' => 1,
                                           'retries' => 1,
                                           'stats' => {
                                                        'success' => 1,
                                                        'retries' => 1,
                                                        'proxiesUsed' => 1,
                                                        'requests' => 1,
                                                        'queries' => 1
                                                      }
                                         },
                               'query' => {
                                            'first' => 'yandex.ru',
                                            'lvl' => 0,
                                            'query' => 'http://yandex.ru',
                                            'queryUid' => '2',
                                            'orig' => 'yandex.ru'
                                          },
                               'pr' => '8',
                               'data' => 'none',
                               'resultsCount' => 1
                             }
                           ]
            };
    Add task: 2463
    Move task: 1
    Task state:
    $VAR1 = {
              'status' => 'completed',
              'stats' => '<b>1. SE::Google::PR</b><br>Queries done: 2<br>Successful
    queries: 2<br>Proxies used: 1 (per query)<br>Retries used: 1 (per query)<br>HTTP
    requests: 1 (per query)',
              'state' => {
                           'totalFail' => 0,
                           'minimized' => 0,
                           'totalWaitProxyThreads' => 0,
                           'queriesDoneCount' => 2,
                           'avgSpeed' => 24,
                           'activeThreads' => 0,
                           'startTime' => 1423653413,
                           'changeTime' => 1423653418,
                           'queriesCount' => 2,
                           'logExists' => 0,
                           'uniqueResultsCount' => 'none',
                           'addTime' => 1423653393,
                           'additionalCount' => 0,
                           'queriesDoneCountAtStart' => 0,
                           'lastQuery' => 'http://yandex.com',
                           'curSpeed' => 40,
                           'started' => 1,
                           'resultsCount' => 2
                         }
            };
    File name: Feb-11_14-16-53.txt at all-api.pl line 26.
    Download url: http://127.0.0.1:9091/downloadResults?fileName=Feb-11_14-16-53.txt&token=leqnjskt
    Delete results file: 1
    Delete task: 1