retriever: Gracefully handle incomplete/broken request files

Apparently the way the dCache ENDIT plugin writes request files can
cause the tsmretriever to barf on an invalid JSON file, for example if
the request file is opened for writing but content isn't written yet.

Work around this by catching errors and retrying a few times with 100ms
delay inbetween.

tsmretriever.pl

@@ -24,6 +24,7 @@
 use JSON;
 use File::Temp qw /tempfile/;
 use File::Basename;
+use Time::HiRes qw(usleep);
 
 # Be noisy when JSON::XS is missing, consider failing hard in the future
 BEGIN {
@@ -80,13 +81,24 @@ ($)
 	my $req_filename = $conf{dir} . '/request/' . $req;
 	my $state;
 
-	{
-		local $/; # slurp whole file
-		# If open failed, probably the request was finished or cancelled
-		open my $rf, '<', $req_filename or return undef;
-		my $json_text = <$rf>;
-		$state = decode_json($json_text);
-		close $rf;
+	return undef unless(-f $req_filename);
+
+	for(1..10) {
+		$state = undef;
+		eval {
+			local $SIG{__WARN__} = sub {};
+			local $SIG{__DIE__} = sub {};
+			local $/; # slurp whole file
+			# If open failed, probably the request was finished or
+			# cancelled.
+			open my $rf, '<', $req_filename or return undef;
+			my $json_text = <$rf>;
+			close $rf;
+			die "Zero-length string" if(length($json_text) == 0);
+			$state = decode_json($json_text);
+		};
+		last if(!$@);
+		usleep(100_000); # Pace ourselves
 	}
 
 	if(!$state || $state->{parent_pid} && getpgrp($state->{parent_pid})<=0)