diff --git a/pom.xml b/pom.xml index 8c81903..9584a2b 100644 --- a/pom.xml +++ b/pom.xml @@ -44,10 +44,9 @@ - org.apache.commons - commons-csv - - + com.opencsv + opencsv + sc.fiji bigdataviewer-core @@ -101,7 +100,7 @@ Mastodon authors Tobias Pietzsch, Jean-Yves Tinevez - 1.0.0-beta-27 + 1.0.0-beta-29 1.0.0-beta-14 diff --git a/src/main/java/org/mastodon/mamut/io/csv/CSVImporter.java b/src/main/java/org/mastodon/mamut/io/csv/CSVImporter.java index 35df785..91ba646 100644 --- a/src/main/java/org/mastodon/mamut/io/csv/CSVImporter.java +++ b/src/main/java/org/mastodon/mamut/io/csv/CSVImporter.java @@ -32,14 +32,11 @@ import java.io.FileReader; import java.io.IOException; import java.io.ObjectInputStream; -import java.io.Reader; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVParser; -import org.apache.commons.csv.CSVRecord; import org.mastodon.RefPool; import org.mastodon.collection.RefCollection; import org.mastodon.feature.Dimension; @@ -60,6 +57,11 @@ import org.scijava.plugin.Plugin; import org.scijava.util.VersionUtils; +import com.opencsv.CSVParser; +import com.opencsv.CSVParserBuilder; +import com.opencsv.CSVReader; +import com.opencsv.CSVReaderBuilder; + import net.imglib2.algorithm.Algorithm; public class CSVImporter implements Algorithm @@ -137,32 +139,36 @@ public static Builder create() @Override public boolean checkInput() { - final CSVFormat csvFormat = CSVFormat.EXCEL - .builder() - .setHeader() - .setCommentMarker( '#' ) - .build(); - try (Reader in = new FileReader( filePath ); - CSVParser records = csvFormat.parse( in );) + if ( separator == '\0' ) { - final Map< String, Integer > headerMap = records.getHeaderMap(); - if ( null == headerMap ) + try { - errorMessage = "File " + filePath + " does not have a header.\n"; - return false; + separator = AutoDetectCSVSeparator.autoDetect( filePath ); + } + catch ( final IOException e1 ) + { + separator = ','; } - } - catch ( final FileNotFoundException e ) + + final CSVParser parser = + new CSVParserBuilder() + .withSeparator( separator ) + .withIgnoreQuotations( true ) + .build(); + try { - errorMessage = "Could not find file " + filePath + "\n" + e.getMessage(); - return false; + new CSVReaderBuilder( new FileReader( filePath ) ) + .withCSVParser( parser ) + .build(); } - catch ( final IOException e ) + catch ( final FileNotFoundException e ) { - errorMessage = "Error reading file " + filePath + "\n" + e.getMessage(); + errorMessage = "Could not find file: " + filePath; + e.printStackTrace(); return false; } + return true; } @@ -185,24 +191,34 @@ public boolean process() } } - final CSVFormat csvFormat = CSVFormat.EXCEL - .builder() - .setDelimiter( separator ) - .setHeader() - .setCommentMarker( '#' ) - .build(); - - try (Reader in = new FileReader( filePath ); - CSVParser records = csvFormat.parse( in );) + final CSVParser parser = + new CSVParserBuilder() + .withSeparator( separator ) + .withIgnoreQuotations( true ) + .build(); + try { + final CSVReader reader = new CSVReaderBuilder( new FileReader( filePath ) ) + .withCSVParser( parser ) + .build(); + final Iterator< String[] > it = reader.iterator(); + + /* + * Parse first line and reads it as the header of the file. + */ + + if ( !it.hasNext() ) + { + errorMessage = "CSV file is empty."; + return false; + } - final Map< String, Integer > uncleanHeaderMap = records.getHeaderMap(); - final Map< String, Integer > headerMap = new HashMap<>( uncleanHeaderMap.size() ); - for ( final String uncleanKey : uncleanHeaderMap.keySet() ) + final String[] firstLine = it.next(); + final Map< String, Integer > headerMap = new HashMap<>( firstLine.length ); + for ( int i = 0; i < firstLine.length; i++ ) { - // Remove control and invisible chars. - final String cleanKey = uncleanKey.trim().replaceAll( "\\p{C}", "" ); - headerMap.put( cleanKey, uncleanHeaderMap.get( uncleanKey ) ); + final String cleanKey = firstLine[ i ].trim().replaceAll( "\\p{C}", "" ); + headerMap.put( cleanKey, Integer.valueOf( i ) ); } /* @@ -264,50 +280,55 @@ public boolean process() labelcol = headerMap.get( labelColumnName ); /* - * Iterate over records. + * Iterate over the rest of lines. */ final WriteLock lock = graph.getLock().writeLock(); lock.lock(); final Spot vref = graph.vertexRef(); final double[] pos = new double[ 3 ]; + try { - for ( final CSVRecord record : records ) + int lineNumber = 1; + while ( it.hasNext() ) { + final String[] record = it.next(); + lineNumber++; + try { - pos[ 0 ] = Double.parseDouble( record.get( xcol ) ) + xOrigin; - pos[ 1 ] = Double.parseDouble( record.get( ycol ) ) + yOrigin; - pos[ 2 ] = Double.parseDouble( record.get( zcol ) ) + zOrigin; - final int t = Integer.parseInt( record.get( framecol ) ); + pos[ 0 ] = Double.parseDouble( record[ xcol ] ) + xOrigin; + pos[ 1 ] = Double.parseDouble( record[ ycol ] ) + yOrigin; + pos[ 2 ] = Double.parseDouble( record[ zcol ] ) + zOrigin; + final int t = Integer.parseInt( record[ framecol ] ); final Spot spot = graph.addVertex( vref ).init( t, pos, radius ); if ( null != idcol ) { - final int id = Integer.parseInt( record.get( idcol ) ); + final int id = Integer.parseInt( record[ idcol ] ); originalIdFeature.set( spot, id ); if ( null == labelcol ) spot.setLabel( "" + id ); } + if ( null != labelcol ) { - spot.setLabel( record.get( labelcol ) ); + spot.setLabel( record[ labelcol ] ); } double q = 1.; if ( null != qualitycol ) { - q = Double.parseDouble( record.get( qualitycol ) ); + q = Double.parseDouble( record[ qualitycol ] ); qualityFeature.set( spot, q ); } } catch ( final NumberFormatException nfe ) { nfe.printStackTrace(); - System.out.println( "Could not parse line " + record.getRecordNumber() + ". Malformed number, skipping.\n" + nfe.getMessage() ); + System.out.println( "Could not parse line " + lineNumber + ". Malformed number, skipping.\n" + nfe.getMessage() ); continue; } - } } finally @@ -318,14 +339,8 @@ public boolean process() } catch ( final FileNotFoundException e ) { + errorMessage = "Cannot find file " + filePath; e.printStackTrace(); - errorMessage = e.getMessage(); - return false; - } - catch ( final IOException e ) - { - e.printStackTrace(); - errorMessage = e.getMessage(); return false; } diff --git a/src/main/java/org/mastodon/mamut/io/csv/plugin/ui/CSVImporterUIController.java b/src/main/java/org/mastodon/mamut/io/csv/plugin/ui/CSVImporterUIController.java index a172d02..94d0f10 100644 --- a/src/main/java/org/mastodon/mamut/io/csv/plugin/ui/CSVImporterUIController.java +++ b/src/main/java/org/mastodon/mamut/io/csv/plugin/ui/CSVImporterUIController.java @@ -32,22 +32,25 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileReader; -import java.io.IOException; -import java.io.Reader; import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; import java.util.Map; import javax.swing.DefaultComboBoxModel; import javax.swing.JComboBox; import javax.swing.JFrame; -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVParser; import org.mastodon.mamut.io.csv.CSVImporter; import org.mastodon.mamut.model.Model; import org.scijava.log.Logger; import org.scijava.log.StderrLogService; +import com.opencsv.CSVParser; +import com.opencsv.CSVParserBuilder; +import com.opencsv.CSVReader; +import com.opencsv.CSVReaderBuilder; + public class CSVImporterUIController { private static final String NONE_COLUMN = "Don't use"; @@ -173,151 +176,147 @@ private boolean readHeaders() * Open and parse file. */ - Reader in; - CSVParser records; - try - { - in = new FileReader( filePath ); - } - catch ( final FileNotFoundException e ) - { - error( "Could not find CSV file:\n" + e.getMessage() + '\n' ); - e.printStackTrace(); - clearComboBoxes(); - return false; - } - + final CSVParser parser = + new CSVParserBuilder() + .withIgnoreQuotations( true ) + .build(); try { - final CSVFormat csvFormat = CSVFormat.EXCEL - .builder() - .setHeader() - .setCommentMarker( '#' ) + final CSVReader reader = new CSVReaderBuilder( new FileReader( filePath ) ) + .withCSVParser( parser ) .build(); - records = csvFormat.parse( in ); - } - catch ( final Exception e ) - { - e.printStackTrace(); - error( "Could not browse CSV file:\n" + e.getMessage() + "\n" ); - clearComboBoxes(); - return false; - } - - this.headerMap = records.getHeaderMap(); - - // Iterate in column orders. - final ArrayList< String > headers = new ArrayList<>( headerMap.keySet() ); - headers.removeIf( ( e ) -> e.trim().isEmpty() ); - - if ( headers.isEmpty() ) - { - error( "Could not read the header of the CSV file.\nIt does not seem present.\n" ); - return false; - } + final Iterator< String[] > it = reader.iterator(); - final String[] mandatory = headers.toArray( new String[] {} ); - view.comboBoxXCol.setModel( new DefaultComboBoxModel<>( mandatory ) ); - view.comboBoxYCol.setModel( new DefaultComboBoxModel<>( mandatory ) ); - view.comboBoxZCol.setModel( new DefaultComboBoxModel<>( mandatory ) ); - view.comboBoxFrameCol.setModel( new DefaultComboBoxModel<>( mandatory ) ); - view.comboBoxTrackCol.setModel( new DefaultComboBoxModel<>( mandatory ) ); - - // Try to be clever and guess from header names. - int tcol = -1; - int xcol = -1; - int ycol = -1; - int zcol = -1; - int trackcol = -1; - for ( int i = 0; i < mandatory.length; i++ ) - { - final String current = mandatory[ i ]; + /* + * Parse first line and reads it as the header of the file. + */ - if ( current.toLowerCase().startsWith( "x" ) || current.toLowerCase().endsWith( "x" ) ) + if ( !it.hasNext() ) { - if ( xcol < 0 || ( current.length() < mandatory[ xcol ].length() ) ) - xcol = i; + error( "CSV file is empty." ); + clearComboBoxes(); + return false; } - if ( current.toLowerCase().startsWith( "y" ) || current.toLowerCase().endsWith( "y" ) ) + final String[] firstLine = it.next(); + this.headerMap = new HashMap<>( firstLine.length ); + for ( int i = 0; i < firstLine.length; i++ ) { - if ( ycol < 0 || ( current.length() < mandatory[ ycol ].length() ) ) - ycol = i; + final String cleanKey = firstLine[ i ].trim().replaceAll( "\\p{C}", "" ); + headerMap.put( cleanKey, Integer.valueOf( i ) ); } - if ( current.toLowerCase().startsWith( "z" ) || current.toLowerCase().endsWith( "z" ) ) - { - if ( zcol < 0 || ( current.length() < mandatory[ zcol ].length() ) ) - zcol = i; - } + // Iterate in column orders. + final ArrayList< String > headers = new ArrayList<>( headerMap.keySet() ); + headers.removeIf( ( e ) -> e.trim().isEmpty() ); - if ( current.toLowerCase().startsWith( "frame" ) - || current.toLowerCase().startsWith( "time" ) - || current.toLowerCase().startsWith( "t" ) ) + if ( headers.isEmpty() ) { - if ( tcol < 0 || current.equals( "frame" ) ) - tcol = i; + error( "Could not read the header of the CSV file.\nIt does not seem present.\n" ); + clearComboBoxes(); + return false; } - if ( current.toLowerCase().startsWith( "track" ) || current.toLowerCase().startsWith( "traj" ) ) + final String[] mandatory = headers.toArray( new String[] {} ); + view.comboBoxXCol.setModel( new DefaultComboBoxModel<>( mandatory ) ); + view.comboBoxYCol.setModel( new DefaultComboBoxModel<>( mandatory ) ); + view.comboBoxZCol.setModel( new DefaultComboBoxModel<>( mandatory ) ); + view.comboBoxFrameCol.setModel( new DefaultComboBoxModel<>( mandatory ) ); + view.comboBoxTrackCol.setModel( new DefaultComboBoxModel<>( mandatory ) ); + + // Try to be clever and guess from header names. + int tcol = -1; + int xcol = -1; + int ycol = -1; + int zcol = -1; + int trackcol = -1; + for ( int i = 0; i < mandatory.length; i++ ) { - if ( trackcol < 0 || current.equals( "track" ) ) - trackcol = i; + final String current = mandatory[ i ]; + + if ( current.toLowerCase().startsWith( "x" ) || current.toLowerCase().endsWith( "x" ) ) + { + if ( xcol < 0 || ( current.length() < mandatory[ xcol ].length() ) ) + xcol = i; + } + + if ( current.toLowerCase().startsWith( "y" ) || current.toLowerCase().endsWith( "y" ) ) + { + if ( ycol < 0 || ( current.length() < mandatory[ ycol ].length() ) ) + ycol = i; + } + + if ( current.toLowerCase().startsWith( "z" ) || current.toLowerCase().endsWith( "z" ) ) + { + if ( zcol < 0 || ( current.length() < mandatory[ zcol ].length() ) ) + zcol = i; + } + + if ( current.toLowerCase().startsWith( "frame" ) + || current.toLowerCase().startsWith( "time" ) + || current.toLowerCase().startsWith( "t" ) ) + { + if ( tcol < 0 || current.equals( "frame" ) ) + tcol = i; + } + + if ( current.toLowerCase().startsWith( "track" ) || current.toLowerCase().startsWith( "traj" ) ) + { + if ( trackcol < 0 || current.equals( "track" ) ) + trackcol = i; + } } - } - if ( tcol < 0 ) - tcol = 0 % ( mandatory.length - 1 ); - if ( xcol < 0 ) - xcol = 1 % ( mandatory.length - 1 ); - if ( ycol < 0 ) - ycol = 2 % ( mandatory.length - 1 ); - if ( zcol < 0 ) - zcol = 3 % ( mandatory.length - 1 ); - if ( trackcol < 0 ) - trackcol = 4 % ( mandatory.length - 1 ); - - view.comboBoxXCol.setSelectedIndex( xcol ); - view.comboBoxYCol.setSelectedIndex( ycol ); - view.comboBoxZCol.setSelectedIndex( zcol ); - view.comboBoxFrameCol.setSelectedIndex( tcol ); - view.comboBoxTrackCol.setSelectedIndex( trackcol ); - - // Add a NONE for non mandatory columns - headers.add( NONE_COLUMN ); - final String[] nonMandatory = headers.toArray( new String[] {} ); - view.comboBoxQualityCol.setModel( new DefaultComboBoxModel<>( nonMandatory ) ); - view.comboBoxNameCol.setModel( new DefaultComboBoxModel<>( nonMandatory ) ); - view.comboBoxIDCol.setModel( new DefaultComboBoxModel<>( nonMandatory ) ); - - int idcol = headers.indexOf( NONE_COLUMN ); - int qualitycol = headers.indexOf( NONE_COLUMN ); - int namecol = headers.indexOf( NONE_COLUMN ); - for ( int i = 0; i < nonMandatory.length; i++ ) - { - final String current = nonMandatory[ i ]; + if ( tcol < 0 ) + tcol = 0 % ( mandatory.length - 1 ); + if ( xcol < 0 ) + xcol = 1 % ( mandatory.length - 1 ); + if ( ycol < 0 ) + ycol = 2 % ( mandatory.length - 1 ); + if ( zcol < 0 ) + zcol = 3 % ( mandatory.length - 1 ); + if ( trackcol < 0 ) + trackcol = 4 % ( mandatory.length - 1 ); + + view.comboBoxXCol.setSelectedIndex( xcol ); + view.comboBoxYCol.setSelectedIndex( ycol ); + view.comboBoxZCol.setSelectedIndex( zcol ); + view.comboBoxFrameCol.setSelectedIndex( tcol ); + view.comboBoxTrackCol.setSelectedIndex( trackcol ); + + // Add a NONE for non mandatory columns + headers.add( NONE_COLUMN ); + final String[] nonMandatory = headers.toArray( new String[] {} ); + view.comboBoxQualityCol.setModel( new DefaultComboBoxModel<>( nonMandatory ) ); + view.comboBoxNameCol.setModel( new DefaultComboBoxModel<>( nonMandatory ) ); + view.comboBoxIDCol.setModel( new DefaultComboBoxModel<>( nonMandatory ) ); + + int idcol = headers.indexOf( NONE_COLUMN ); + int qualitycol = headers.indexOf( NONE_COLUMN ); + int namecol = headers.indexOf( NONE_COLUMN ); + for ( int i = 0; i < nonMandatory.length; i++ ) + { + final String current = nonMandatory[ i ]; - if ( current.toLowerCase().startsWith( "id" ) - || current.toLowerCase().startsWith( "index" ) ) - idcol = i; + if ( current.toLowerCase().startsWith( "id" ) + || current.toLowerCase().startsWith( "index" ) ) + idcol = i; - if ( current.toLowerCase().startsWith( "name" ) ) - namecol = i; + if ( current.toLowerCase().startsWith( "name" ) ) + namecol = i; - if ( current.toLowerCase().startsWith( "q" ) ) - qualitycol = i; - } + if ( current.toLowerCase().startsWith( "q" ) ) + qualitycol = i; + } - view.comboBoxIDCol.setSelectedIndex( idcol ); - view.comboBoxQualityCol.setSelectedIndex( qualitycol ); - view.comboBoxNameCol.setSelectedIndex( namecol ); - try - { - in.close(); + view.comboBoxIDCol.setSelectedIndex( idcol ); + view.comboBoxQualityCol.setSelectedIndex( qualitycol ); + view.comboBoxNameCol.setSelectedIndex( namecol ); } - catch ( final IOException e ) + catch ( final FileNotFoundException e ) { - error( "Problem closing the CSV file:\n" + e.getMessage() + '\n' ); + error( "Cannot find file " + filePath ); e.printStackTrace(); + return false; } return true; } diff --git a/src/test/java/org/mastodon/mamut/io/csv/CSVImporterTestDrive.java b/src/test/java/org/mastodon/mamut/io/csv/CSVImporterTestDrive.java index 0f0a6ad..773f632 100644 --- a/src/test/java/org/mastodon/mamut/io/csv/CSVImporterTestDrive.java +++ b/src/test/java/org/mastodon/mamut/io/csv/CSVImporterTestDrive.java @@ -51,21 +51,26 @@ public static void main( final String[] args ) throws IOException, SpimDataExcep Locale.setDefault( Locale.ROOT ); UIManager.setLookAndFeel( UIManager.getSystemLookAndFeelClassName() ); - final String bdvFile = "samples/200212__pos1.xml"; +// final String bdvFile = "samples/200212__pos1.xml"; + final String bdvFile = "../mastodon/samples/datasethdf5.xml"; final ProjectModel projectModel = ProjectCreator.createProjectFromBdvFile( new File( bdvFile ), new Context() ); new MainWindow( projectModel ).setVisible( true ); final Model model = projectModel.getModel(); - final String csvFilePath = "samples/200212__pos1.csv"; +// final String csvFilePath = "samples/200212__pos1.csv"; + final String csvFilePath = "samples/MastodonTable-Spot-1lineheader.csv"; final CSVImporter importer = CSVImporter.create() .model( model ) .csvFilePath( csvFilePath ) + .separator( ',' ) .radius( 2. ) - .xColumnName( "x" ) - .yColumnName( "y" ) - .zColumnName( "z" ) - .frameColumnName( "time" ) - .idColumnName( "index" ) + .xColumnName( "X" ) + .yColumnName( "Y" ) + .zColumnName( "Z" ) + .frameColumnName( "Spot frame" ) + .idColumnName( "ID" ) + .labelColumnName( "Label" ) + .qualityColumnName( "Detection quality" ) .get(); System.out.println( "Starting import" ); diff --git a/src/test/java/org/mastodon/mamut/io/csv/CSVImporterUITestDrive.java b/src/test/java/org/mastodon/mamut/io/csv/CSVImporterUITestDrive.java index e73d48c..9f58266 100644 --- a/src/test/java/org/mastodon/mamut/io/csv/CSVImporterUITestDrive.java +++ b/src/test/java/org/mastodon/mamut/io/csv/CSVImporterUITestDrive.java @@ -53,11 +53,12 @@ public static void main( final String[] args ) throws IOException, SpimDataExcep Locale.setDefault( Locale.ROOT ); UIManager.setLookAndFeel( UIManager.getSystemLookAndFeelClassName() ); - final String bdvFile = "samples/200212__pos1.xml"; +// final String bdvFile = "samples/200212__pos1.xml"; + final String bdvFile = "../mastodon/samples/datasethdf5.xml"; final ProjectModel projectModel = ProjectCreator.createProjectFromBdvFile( new File( bdvFile ), new Context() ); new MainWindow( projectModel ).setVisible( true ); - final String csvFilePath = "samples/200212__pos1.csv"; + final String csvFilePath = "samples/MastodonTable-Spot-1lineheader.csv"; final ToggleCSVImporterDialogAction action = ( ToggleCSVImporterDialogAction ) projectModel .getPlugins() .getPluginActions()