Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rework the CSV importer: use opencsv as deps. #13

Merged
merged 3 commits into from
May 3, 2024
Merged

Rework the CSV importer: use opencsv as deps. #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4ec2907
Update mastodon dep version.
tinevez May 3, 2024
99f0811
Rework the CSV importer to use opencsv.
tinevez May 3, 2024
e59b15c
Tweak the test drives.
tinevez May 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 4 additions & 5 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,9 @@

<!-- Other deps -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
</dependency>

<groupId>com.opencsv</groupId>
<artifactId>opencsv</artifactId>
</dependency>
<dependency>
<groupId>sc.fiji</groupId>
<artifactId>bigdataviewer-core</artifactId>
Expand Down Expand Up @@ -101,7 +100,7 @@
<license.organizationName>Mastodon authors</license.organizationName>
<license.copyrightOwners>Tobias Pietzsch, Jean-Yves Tinevez</license.copyrightOwners>

<mastodon.version>1.0.0-beta-27</mastodon.version>
<mastodon.version>1.0.0-beta-29</mastodon.version>
<mastodon-tracking.version>1.0.0-beta-14</mastodon-tracking.version>

<!-- NB: Deploy releases to the SciJava Maven repository. -->
Expand Down
123 changes: 69 additions & 54 deletions src/main/java/org/mastodon/mamut/io/csv/CSVImporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,11 @@
import java.io.FileReader;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.Reader;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;

import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.mastodon.RefPool;
import org.mastodon.collection.RefCollection;
import org.mastodon.feature.Dimension;
Expand All @@ -60,6 +57,11 @@
import org.scijava.plugin.Plugin;
import org.scijava.util.VersionUtils;

import com.opencsv.CSVParser;
import com.opencsv.CSVParserBuilder;
import com.opencsv.CSVReader;
import com.opencsv.CSVReaderBuilder;

import net.imglib2.algorithm.Algorithm;

public class CSVImporter implements Algorithm
Expand Down Expand Up @@ -137,32 +139,36 @@ public static Builder create()
@Override
public boolean checkInput()
{
final CSVFormat csvFormat = CSVFormat.EXCEL
.builder()
.setHeader()
.setCommentMarker( '#' )
.build();
try (Reader in = new FileReader( filePath );
CSVParser records = csvFormat.parse( in );)
if ( separator == '\0' )
{
final Map< String, Integer > headerMap = records.getHeaderMap();
if ( null == headerMap )
try
{
errorMessage = "File " + filePath + " does not have a header.\n";
return false;
separator = AutoDetectCSVSeparator.autoDetect( filePath );
}
catch ( final IOException e1 )
{
separator = ',';
}

}
catch ( final FileNotFoundException e )

final CSVParser parser =
new CSVParserBuilder()
.withSeparator( separator )
.withIgnoreQuotations( true )
.build();
try
{
errorMessage = "Could not find file " + filePath + "\n" + e.getMessage();
return false;
new CSVReaderBuilder( new FileReader( filePath ) )
.withCSVParser( parser )
.build();
}
catch ( final IOException e )
catch ( final FileNotFoundException e )
{
errorMessage = "Error reading file " + filePath + "\n" + e.getMessage();
errorMessage = "Could not find file: " + filePath;
e.printStackTrace();
return false;
}

return true;
}

Expand All @@ -185,24 +191,34 @@ public boolean process()
}
}

final CSVFormat csvFormat = CSVFormat.EXCEL
.builder()
.setDelimiter( separator )
.setHeader()
.setCommentMarker( '#' )
.build();

try (Reader in = new FileReader( filePath );
CSVParser records = csvFormat.parse( in );)
final CSVParser parser =
new CSVParserBuilder()
.withSeparator( separator )
.withIgnoreQuotations( true )
.build();
try
{
final CSVReader reader = new CSVReaderBuilder( new FileReader( filePath ) )
.withCSVParser( parser )
.build();
final Iterator< String[] > it = reader.iterator();

/*
* Parse first line and reads it as the header of the file.
*/

if ( !it.hasNext() )
{
errorMessage = "CSV file is empty.";
return false;
}

final Map< String, Integer > uncleanHeaderMap = records.getHeaderMap();
final Map< String, Integer > headerMap = new HashMap<>( uncleanHeaderMap.size() );
for ( final String uncleanKey : uncleanHeaderMap.keySet() )
final String[] firstLine = it.next();
final Map< String, Integer > headerMap = new HashMap<>( firstLine.length );
for ( int i = 0; i < firstLine.length; i++ )
{
// Remove control and invisible chars.
final String cleanKey = uncleanKey.trim().replaceAll( "\\p{C}", "" );
headerMap.put( cleanKey, uncleanHeaderMap.get( uncleanKey ) );
final String cleanKey = firstLine[ i ].trim().replaceAll( "\\p{C}", "" );
headerMap.put( cleanKey, Integer.valueOf( i ) );
}

/*
Expand Down Expand Up @@ -264,50 +280,55 @@ public boolean process()
labelcol = headerMap.get( labelColumnName );

/*
* Iterate over records.
* Iterate over the rest of lines.
*/

final WriteLock lock = graph.getLock().writeLock();
lock.lock();
final Spot vref = graph.vertexRef();
final double[] pos = new double[ 3 ];

try
{
for ( final CSVRecord record : records )
int lineNumber = 1;
while ( it.hasNext() )
{
final String[] record = it.next();
lineNumber++;

try
{
pos[ 0 ] = Double.parseDouble( record.get( xcol ) ) + xOrigin;
pos[ 1 ] = Double.parseDouble( record.get( ycol ) ) + yOrigin;
pos[ 2 ] = Double.parseDouble( record.get( zcol ) ) + zOrigin;
final int t = Integer.parseInt( record.get( framecol ) );
pos[ 0 ] = Double.parseDouble( record[ xcol ] ) + xOrigin;
pos[ 1 ] = Double.parseDouble( record[ ycol ] ) + yOrigin;
pos[ 2 ] = Double.parseDouble( record[ zcol ] ) + zOrigin;
final int t = Integer.parseInt( record[ framecol ] );

final Spot spot = graph.addVertex( vref ).init( t, pos, radius );
if ( null != idcol )
{
final int id = Integer.parseInt( record.get( idcol ) );
final int id = Integer.parseInt( record[ idcol ] );
originalIdFeature.set( spot, id );
if ( null == labelcol )
spot.setLabel( "" + id );
}

if ( null != labelcol )
{
spot.setLabel( record.get( labelcol ) );
spot.setLabel( record[ labelcol ] );
}
double q = 1.;
if ( null != qualitycol )
{
q = Double.parseDouble( record.get( qualitycol ) );
q = Double.parseDouble( record[ qualitycol ] );
qualityFeature.set( spot, q );
}
}
catch ( final NumberFormatException nfe )
{
nfe.printStackTrace();
System.out.println( "Could not parse line " + record.getRecordNumber() + ". Malformed number, skipping.\n" + nfe.getMessage() );
System.out.println( "Could not parse line " + lineNumber + ". Malformed number, skipping.\n" + nfe.getMessage() );
continue;
}

}
}
finally
Expand All @@ -318,14 +339,8 @@ public boolean process()
}
catch ( final FileNotFoundException e )
{
errorMessage = "Cannot find file " + filePath;
e.printStackTrace();
errorMessage = e.getMessage();
return false;
}
catch ( final IOException e )
{
e.printStackTrace();
errorMessage = e.getMessage();
return false;
}

Expand Down
Loading
Loading