Pulling data from API, memory growth

I’m working on a project where I pull data (JSON) from an API. The problem I’m having is that the memory is slowly growing untill I get the dreaded fatal error:

Fatal error: Allowed memory size of * bytes exhausted (tried
to allocate * bytes) in C:… on line *

I don’t think there should be any memory growth. I tried unsetting everything at the end of the loop but no difference. So my question is: am I doing something wrong? Is it normal? What can I do to fix this problem?

<?php

include ('connect.php');    
set_time_limit(0);

$start = microtime(true);    
$api_key = '***';
$threads = 10; //number of urls called simultaneuosly

function multiRequest($urls) {

    $nbrURLS = count($urls); // number of urls in array $urls
    $ch = array(); // array of curl handles
    $result = array(); // data to be returned

    $mh = curl_multi_init(); // create a multi handle 


    // set URL and other appropriate options
    for($i = 0; $i < $nbrURLS; $i++) {
        $ch[$i]=curl_init();

        curl_setopt($ch[$i], CURLOPT_URL, $urls[$i]); // set URL
        curl_setopt($ch[$i], CURLOPT_RETURNTRANSFER, 1); // return data as string
        curl_setopt($ch[$i], CURLOPT_SSL_VERIFYPEER, 0); // Doesn't verifies certificate

        curl_multi_add_handle ($mh, $ch[$i]); // Add a normal cURL handle to a cURL multi handle
    }

    // execute the handles
    do {
        $mrc = curl_multi_exec($mh, $active);          
        curl_multi_select($mh, 0.1);
    } while ($active);

    // get content and remove handles
    for($i = 0; $i < $nbrURLS; $i++) {

         $error = curl_getinfo($ch[$i], CURLINFO_HTTP_CODE); // Last received HTTP code 

        //error handling if not 200 ok code
        if($error != 200){

            if($error == 429 || $error == 500 || $error == 503 || $error == 504){
                $result['again'][] = $urls[$i];

            } else {
                $result['errors'][] = array("Url" => $urls[$i], "errornbr" => $error);
            }

        } else {
            $result['json'][] = curl_multi_getcontent($ch[$i]);
        }

        curl_multi_remove_handle($mh, $ch[$i]);
        curl_close($ch[$i]);
    }

    curl_multi_close($mh);

    return $result;
}


$id = mysqli_query($connect, "SELECT id FROM `apiTable` LIMIT 1000 ");
$urls = array();

while($result = mysqli_fetch_array($id))
{
      $urls[] = 'https://website.com/api/json/' . $ids['id'] . '?api_key=' . $api_key;
}

$x = 1; //number of loops

while($urls){ 

    $chunk = array_splice($urls, 0, $threads); // take the first chunk ($threads) of all urls
    $result = multiRequest($chunk); // Get json

    unset($chunk);

    $nbrComplete = count($result['json']); //number of retruned json strings

    for($y = 0; $y < $nbrComplete; $y++){        
        $decoded = json_decode($result['json'][$y], true);// parse the json

        /////////////
        // manipulate all the data and store
        /////////////

    }

    unset($nbrComplete);
    unset($decoded);

    $time = microtime(true) - $start;
    echo $x . ": ". memory_get_peak_usage(true) . " | " . $time . "<br>";

    // reuse urls
    if(isset($result['again'])){
        $urls = array_merge($urls, $result['again']);
        unset($result['again']);
    }

    unset($result);
    unset($time);

    sleep(15); // limit the request rate

    $x++;
}
?>

100 loops:

loop: memory | time (sec)
1: 5505024 | 0.98330211639404
3: 6291456 | 33.190237045288
65: 6553600 | 1032.1401019096
73: 6815744 | 1160.4345710278
75: 7077888 | 1192.6274609566
100: 7077888 | 1595.2397520542
Advertisements

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s