Infrastructure

# a quick script to look at the uniqueness of identifiers returned from a node.

# It uses the start and count parameters to get the full object list in 

# successive calls to the node.

$NODE = "https://cn-orc-1.dataone.org/cn";

$pageSize = 10000;

($response) = listObjects($NODE,0,0);

($total) = $response=~/total="(\d+)"/;

print "total objects for $NODE : $total\n";

$runningTotal = 0;

while ($runningTotal < $total) {

    $response = listObjects($NODE,$runningTotal,$pageSize);

    @oinfos = split("<objectInfo>",$response);

    $header = shift @oinfos;

    print STDERR "**running total = $runningTotal\n" if ($runningTotal % 10000 == 0);

    foreach (@oinfos) {

	my ($id) = /<identifier>(.+)<\/identifier>/;

	if ($id) {

	    $runningTotal++;

	    $idSet{$id}++;

	} else {

	    print "$_\n";

	}

    }

}

print "number of unique identifiers: " . scalar(keys %idSet) . "\n";

foreach (sort keys %idSet) {

    if ($idSet{$_} > 1) {

	print "  $_ : $idSet{$_}\n";

    }

}

sub listObjects {

    my($baseUrl,$start,$count) = @_;

    my $cmd = "curl '$baseUrl/v1/object?start=$start&count=$count'";

    return `$cmd`; 

}